Prepare for 0.4 release

CHANGELOG.md

@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 
-## Upcoming [0.4.0] (~August 2026)
+## [0.4.0] - 2026-09-03
 
 ### Added
 * **Added support for streaming data read of remote NWB files:**
@@ -74,6 +74,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
    * Updated unit tests to create `Subject` for all NWB files generated in the test suite to comply with NWB best practices. `nwbinspector` tests require subject by default now. (@oruebel, [#289](https://github.com/NeurodataWithoutBorders/aqnwb/pull/289) removed this requirements by specifying  `--ignore=check_subject_exists` and [#320](https://github.com/NeurodataWithoutBorders/aqnwb/pull/320) removed the ignore to ensure that subject is always present in the test files)
 
 ### Fixed
+* Fixed overly restrictive ISO 8601 datetime validation in `isISO8601Date()` that incorrectly rejected valid timestamps with optional fractional seconds and UTC (`Z`), basic, or extended numeric timezone offsets. (@copilot, @oruebel, [#340](https://github.com/NeurodataWithoutBorders/aqnwb/pull/340))
 * Fixed `get_utc_offset_seconds` to correctly account for daylight saving time using platform-specific APIs (`tm_gmtoff` on Unix/macOS; `_get_timezone` + `_get_dstbias` on Windows), preventing `session_start_time` from being written ~1 hour ahead of UTC during DST (@cboulay, [#295](https://github.com/NeurodataWithoutBorders/aqnwb/pull/295))
 * Fixed `resources/utils/schematype_to_aqnwb.py` to facilitate generation of new types:
     * Emit forward declarations rather than full header includes for referenced generated types, avoiding circular header dependencies. This issue surfaced due to the update to the latest HDMF Common schema types where `DynamicTable` references `MeaningsTable`, which is itself a `DynamicTable` (@copilot, @oruebel, [#300](https://github.com/NeurodataWithoutBorders/aqnwb/pull/300))

CMakeLists.txt

@@ -8,7 +8,7 @@ include(cmake/prelude.cmake)
 
 project(
     aqnwb
-    VERSION 0.3.0
+    VERSION 0.4.0
     DESCRIPTION "An API to acquire neurophysiology data in the NWB format"
     LANGUAGES CXX
 )

src/Utils.hpp

@@ -191,19 +191,20 @@ static inline std::string getCurrentTime()
 }
 
 /**
- * @brief Check that a string is formatted in ISO8601 format
+ * @brief Check that a string is formatted as an ISO 8601 datetime.
  *
- * This function only validates the regex pattern but does not check that
- * the time values specified are indeed valid.
+ * This function only validates the datetime syntax and requires a timezone
+ * designator. It does not check that the specified date and time values are
+ * valid.
  *
  * @return bool indicating whether the string is in ISO8601 form
  */
 static inline bool isISO8601Date(const std::string& dateStr)
 {
-  // Define the regex pattern for ISO 8601 extended format with timezone offset
-  // Allow one or more fractional seconds digits
+  // Require extended date/time fields and a UTC or numeric timezone designator.
+  // Fractional seconds and the colon in numeric offsets are optional.
   const std::string iso8601Pattern =
-      R"(^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{2}:\d{2}$)";
+      R"(^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})$)";
   std::regex pattern(iso8601Pattern);
 
   // Check if the date string matches the regex pattern

src/nwb/hdmf/table/DynamicTable.cpp

@@ -427,6 +427,23 @@ Status DynamicTable::addRows(const std::vector<AQNWB::Types::RowData>& rows,
 
   // 2. Validate that all rows have the required columns
   for (const auto& row : rows) {
+    // 2.1 Validate the keys in the row, e.g., no unknown columns
+    for (const auto& [key, value] : row) {
+      if (key == "id") {
+        std::cerr << "DynamicTable::addRows: Row IDs must be provided via the "
+                     "rowIds parameter, not as an 'id' key in the row data."
+                  << std::endl;
+        return Status::Failure;
+      }
+      if (m_configuredColumnIndices.find(key) == m_configuredColumnIndices.end()
+          && targetColumns.find(key) == targetColumns.end())
+      {
+        std::cerr << "DynamicTable::addRows: Unknown column '" << key
+                  << "' provided." << std::endl;
+        return Status::Failure;
+      }
+    }
+    // 2.2 Validate against the configured columns
     for (const auto& col : m_configuredColumns) {
       if (targetColumns.find(col.name) != targetColumns.end()) {
         // Target columns must be present in the row
@@ -458,11 +475,20 @@ Status DynamicTable::addRows(const std::vector<AQNWB::Types::RowData>& rows,
     }
   }
 
-  // 3. Buffer values for each column
+  // 3. Buffer values for each column.
+  // We accumulate all the incoming row data into memory buffers before
+  // performing any disk I/O. This batching strategy significantly reduces I/O
+  // overhead and helps ensure consistency: if a type mismatch or validation
+  // error is found in the incoming data, we can fail early without leaving the
+  // file partially modified.
   std::unordered_map<std::string, IO::BaseDataType::BaseDataVectorVariant>
       columnBuffers;
-  std::unordered_map<std::string, std::vector<uint32_t>> indexBuffers;
+  std::unordered_map<std::string, IO::BaseDataType::BaseDataVectorVariant>
+      indexBuffers;
 
+  // Initialize empty variants with the correct underlying vector type (e.g.
+  // std::vector<float>) based on the column's data type, so that std::visit can
+  // append to them later.
   for (const auto& col : m_configuredColumns) {
     auto vectorIndex = std::dynamic_pointer_cast<VectorIndex>(col.column);
     if (vectorIndex) {
@@ -472,13 +498,16 @@ Status DynamicTable::addRows(const std::vector<AQNWB::Types::RowData>& rows,
       std::string targetName = target->getName();
       columnBuffers[targetName] = IO::BaseDataType::createEmptyVectorVariant(
           target->readData()->getDataType());
-      indexBuffers[col.name] = std::vector<uint32_t>();
+      indexBuffers[col.name] = IO::BaseDataType::createEmptyVectorVariant(
+          vectorIndex->readData()->getDataType());
     } else if (targetColumns.find(col.name) == targetColumns.end()) {
       columnBuffers[col.name] =
           IO::BaseDataType::createEmptyVectorVariant(col.dataType);
     }
   }
 
+  // Iterate over each user-provided row and serialize their cell values into
+  // our memory buffers
   for (const auto& row : rows) {
     for (const auto& col : m_configuredColumns) {
       auto vectorIndex = std::dynamic_pointer_cast<VectorIndex>(col.column);
@@ -489,7 +518,9 @@ Status DynamicTable::addRows(const std::vector<AQNWB::Types::RowData>& rows,
         std::string targetName = target->getName();
         auto it = row.find(targetName);
         if (it != row.end()) {
-          // Also need to account for existing elements in the target column
+          // For ragged columns, we need to track how many elements exist in the
+          // target dataset so we know what the boundary index is for the
+          // current ragged element.
           size_t existingTargetElements = 0;
           auto targetData = target->readData();
           if (targetData) {
@@ -519,9 +550,40 @@ Status DynamicTable::addRows(const std::vector<AQNWB::Types::RowData>& rows,
               },
               columnBuffers[targetName]);
 
-          uint32_t newIndex =
-              static_cast<uint32_t>(existingTargetElements + newTargetSize);
-          indexBuffers[col.name].push_back(newIndex);
+          size_t newIndex = existingTargetElements + newTargetSize;
+          Status indexAppendStatus = std::visit(
+              [newIndex, &col](auto& vec) -> Status
+              {
+                using VecType = std::decay_t<decltype(vec)>;
+                if constexpr (std::is_same_v<VecType, std::monostate>) {
+                  return Status::Failure;
+                } else {
+                  using ValType = typename VecType::value_type;
+                  // std::visit forces the compiler to generate code for ALL
+                  // possible types in BaseDataVectorVariant (including
+                  // std::vector<std::string>). We use if constexpr to prevent
+                  // compilation errors when casting size_t to a string.
+                  // VectorIndex must be an integer, so we reject other types.
+                  // Note: HDMF formally requires VectorIndex to be unsigned.
+                  // However, we use std::is_integral_v to be lenient and permit
+                  // signed integers as well, in case we are appending to an
+                  // existing non-strict dataset.
+                  if constexpr (std::is_integral_v<ValType>) {
+                    vec.push_back(static_cast<ValType>(newIndex));
+                    return Status::Success;
+                  } else {
+                    std::cerr
+                        << "DynamicTable::addRows: VectorIndex '" << col.name
+                        << "' has invalid non-integral data type." << std::endl;
+                    return Status::Failure;
+                  }
+                }
+              },
+              indexBuffers[col.name]);
+
+          if (indexAppendStatus != Status::Success) {
+            return Status::Failure;
+          }
         }
       } else if (targetColumns.find(col.name) == targetColumns.end()) {
         // Regular column (not a target of a VectorIndex)
@@ -549,6 +611,17 @@ Status DynamicTable::addRows(const std::vector<AQNWB::Types::RowData>& rows,
         return Status::Failure;
       std::string targetName = target->getName();
 
+      // Write index buffer
+      IO::BaseDataType::BaseDataVectorVariant indexVariant =
+          indexBuffers[col.name];
+      Status writeIndexStatus = vectorIndex->appendBuffer(indexVariant);
+      if (writeIndexStatus != Status::Success) {
+        std::cerr << "DynamicTable::addRows: Failed to write buffer for index "
+                     "column '"
+                  << col.name << "'." << std::endl;
+        return Status::Failure;
+      }
+
       // Write target buffer
       if (columnBuffers.find(targetName) != columnBuffers.end()) {
         Status writeStatus = target->appendBuffer(columnBuffers[targetName]);
@@ -562,17 +635,6 @@ Status DynamicTable::addRows(const std::vector<AQNWB::Types::RowData>& rows,
         // it
         columnBuffers.erase(targetName);
       }
-
-      // Write index buffer
-      IO::BaseDataType::BaseDataVectorVariant indexVariant =
-          indexBuffers[col.name];
-      Status writeIndexStatus = vectorIndex->appendBuffer(indexVariant);
-      if (writeIndexStatus != Status::Success) {
-        std::cerr << "DynamicTable::addRows: Failed to write buffer for index "
-                     "column '"
-                  << col.name << "'." << std::endl;
-        return Status::Failure;
-      }
     } else if (targetColumns.find(col.name) == targetColumns.end()) {
       // Write regular column buffer
       Status writeStatus = col.column->appendBuffer(columnBuffers[col.name]);
@@ -891,6 +953,20 @@ Status DynamicTable::loadConfiguredColumnsFromFile()
     auto col = readColumn<VectorData>(colName);
     if (col) {
       registerColumn(col);
+
+      // Also check for associated index columns because m_colNames only lists
+      // target columns. A ragged array of ragged arrays would have multiple
+      // "_index" suffixes.
+      std::string currentIndexName = colName + "_index";
+      while (true) {
+        auto indexCol = readColumn<VectorIndex>(currentIndexName);
+        if (indexCol) {
+          registerColumn(indexCol);
+          currentIndexName += "_index";
+        } else {
+          break;
+        }
+      }
     }
   }
 

src/nwb/hdmf/table/VectorIndex.cpp

@@ -197,19 +197,11 @@ AQNWB::Types::CellValue VectorIndex::combineCellsToVector(
 }
 
 std::vector<AQNWB::Types::CellValue> VectorIndex::readIndexedCellValues(
-    SizeType start, SizeType count, SizeType stride, SizeType block)
+    SizeType start, SizeType count)
 {
   if (count == 0) {
     return {};
   }
-  if (stride == 0) {
-    throw std::invalid_argument(
-        "VectorIndex::readIndexedCellValues: stride must be greater than 0");
-  }
-  if (block == 0) {
-    throw std::invalid_argument(
-        "VectorIndex::readIndexedCellValues: block must be greater than 0");
-  }
 
   auto ioPtr = getIO();
   if (!ioPtr || !ioPtr->isOpen()) {
@@ -229,11 +221,9 @@ std::vector<AQNWB::Types::CellValue> VectorIndex::readIndexedCellValues(
   SizeArray startArray = {start};
   SizeArray countArray =
       count != AQNWB::Types::SizeTypeNotSet ? SizeArray {count} : SizeArray {};
-  SizeArray strideArray = {stride};
-  SizeArray blockArray = {block};
 
-  std::vector<AQNWB::Types::CellValue> indices = VectorData::readCellValues(
-      startArray, countArray, strideArray, blockArray);
+  std::vector<AQNWB::Types::CellValue> indices =
+      VectorData::readCellValues(startArray, countArray);
 
   if (indices.empty()) {
     return {};

src/nwb/hdmf/table/VectorIndex.hpp

@@ -136,8 +136,6 @@ class VectorIndex : public AQNWB::NWB::VectorData
    *
    * @param start The starting index for the slice (optional).
    * @param count The number of elements to read (optional).
-   * @param stride The stride (optional).
-   * @param block The block size (optional).
    * @return A vector of CellValue, where each CellValue contains the data for a
    * cell.
    * @throws std::out_of_range if the requested range is invalid (e.g., start +
@@ -149,10 +147,7 @@ class VectorIndex : public AQNWB::NWB::VectorData
    * invalid (e.g., not monotonically increasing).
    */
   std::vector<AQNWB::Types::CellValue> readIndexedCellValues(
-      SizeType start = 0,
-      SizeType count = AQNWB::Types::SizeTypeNotSet,
-      SizeType stride = 1,
-      SizeType block = 1);
+      SizeType start = 0, SizeType count = AQNWB::Types::SizeTypeNotSet);
 
   // Define read methods
   DEFINE_REFERENCED_REGISTERED_FIELD(