Pip install for tools

.github/workflows/generate-classes.yml

@@ -40,10 +40,8 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Install python script dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r resources/utils/requirements.txt
+      - name: Install uv
+        run: brew install uv
 
 # -----------------------------------------------------------------
 # 2. LabMetadataExtension pipeline
@@ -52,11 +50,11 @@ jobs:
         run: |
           mkdir test_output
           mkdir test_output/spec
-          python resources/utils/generate_spec_files.py demo/labmetadata_extension_demo/spec test_output/spec
+          uv run resources/utils/aqnwb_utils.py generate-spec demo/labmetadata_extension_demo/spec test_output/spec
 
       - name: Run schematype_to_aqnwb.py to generate AqNWB classes for the LabMetadataExample extension
         run: |
-          python resources/utils/schematype_to_aqnwb.py --generate-test-app demo/labmetadata_extension_demo/spec/ndx-labmetadata-example.namespace.yaml test_output
+          uv run resources/utils/aqnwb_utils.py generate-types --generate-test-app demo/labmetadata_extension_demo/spec/ndx-labmetadata-example.namespace.yaml test_output
 
       - name: List generated files for the LabMetadataExample extension
         run: | 
@@ -67,9 +65,8 @@ jobs:
           cd test_output/test_app
           mkdir build
           cd build
-          cmake -DAQNWB_DIR="${{ github.workspace }}/build" ../
+          cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" ../
           make -j 2
-          ./bin/schema_compilation_test
 
       - name: Run the test program for LabMetadataExample extension
         run: |
@@ -96,11 +93,11 @@ jobs:
       - name: Run generate_nwb_schema_headers.sh to generated headers the nwb_schema
         run: |
           mkdir test_output_nwb_schema/spec
-          python resources/utils/generate_spec_files.py test_output_nwb_schema/nwb-schema/core test_output_nwb_schema/spec
+          uv run resources/utils/aqnwb_utils.py generate-spec test_output_nwb_schema/nwb-schema/core test_output_nwb_schema/spec
 
       - name: Run schematype_to_aqnwb.py to generate AqNWB classes for the nwb_schema
         run: |
-          python resources/utils/schematype_to_aqnwb.py --generate-test-app test_output_nwb_schema/nwb-schema/core/nwb.namespace.yaml test_output_nwb_schema
+          uv run resources/utils/aqnwb_utils.py generate-types --generate-test-app test_output_nwb_schema/nwb-schema/core/nwb.namespace.yaml test_output_nwb_schema
 
       - name: List generated files for the nwb-schema
         run: |
@@ -111,12 +108,10 @@ jobs:
           cd test_output_nwb_schema/test_app
           mkdir build
           cd build
-          cmake -DAQNWB_DIR="${{ github.workspace }}/build" ../
+          cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" ../
           make -j 2
 
       - name: Run the test program for the nwb_schema
         run: |
           cd test_output_nwb_schema/test_app/build
           ./bin/schema_compilation_test
-
-

.github/workflows/python-utils.yml

@@ -19,10 +19,8 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r resources/utils/requirements.txt
+      - name: Install uv
+        run: brew install uv
 
       - name: Clone latest NWB schema release
         run: |
@@ -37,17 +35,17 @@ jobs:
           echo "Checking out NWB schema at tag: $NWB_LATEST_TAG"
           git checkout "$NWB_LATEST_TAG"
 
-      - name: Run generate_spec_files.py
+      - name: Run generate_spec_files
         run: |
-          output=$(python resources/utils/generate_spec_files.py test_output/nwb-schema/core test_output 2>&1)
+          output=$(uv run resources/utils/aqnwb_utils.py generate-spec test_output/nwb-schema/core test_output 2>&1)
           echo "$output"
           if echo "$output" | grep -q "ERROR"; then
             exit 1
           fi
 
-      - name: Run schematype_to_aqnwb.py
+      - name: Run schematype_to_aqnwb
         run: |
-          output=$(python resources/utils/schematype_to_aqnwb.py test_output/nwb-schema/core/nwb.namespace.yaml test_output 2>&1)
+          output=$(uv run resources/utils/aqnwb_utils.py generate-types test_output/nwb-schema/core/nwb.namespace.yaml test_output 2>&1)
           echo "$output"
           if echo "$output" | grep -q "ERROR"; then
             exit 1

.gitignore

@@ -30,3 +30,23 @@ demo/cmake-build-*/
 demo/inspect_electrical_series/*.nwb
 demo/*/Makefile
 demo/*/cmake_install.cmake
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.Python
+env/
+venv/
+.ipynb_checkpoints
+
+# Python packaging
+build/
+dist/
+*.egg-info/
+wheels/
+
+# Pytest
+.pytest_cache/
+htmlcov/

CHANGELOG.md

@@ -7,7 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+* Python Utility enhancements:
+    * Added `aqnwb-utils` as a command-line utility to provide a common interface for aqnwb command line tools, e.g., `schematype_to_aqnwb.py` and `generate_spec_files.py`. (@oruebel, [#227](https://github.com/NeurodataWithoutBorders/aqnwb/pull/227))
+    * Added inline script metadata (PEP 723) to Python utilities to enable direct execution with `uv run` without installation (@oruebel, [#229](https://github.com/NeurodataWithoutBorders/aqnwb/pull/229)
+    * Added `pyproject.toml` for modern Python packaging support (@oruebel, [#229](https://github.com/NeurodataWithoutBorders/aqnwb/pull/229)
+
 ### Changed
+* Updated Python utilities to use `uv` instead of `pip` for dependency management and updated docs and github workflows to use uv (@oruebel, [#227](https://github.com/NeurodataWithoutBorders/aqnwb/pull/227)
+* Updated documentation to refer to the new `aqnwb-utils` command-line utility (@oruebel, [#227](https://github.com/NeurodataWithoutBorders/aqnwb/pull/227))
 * Enhanced the `schematype_to_aqnwb` utility script:
     * Generated source files are now placed into a folder hierarchy based on the name of the namespace and schemafile of the neurodata_type (@oruebel, [#224](https://github.com/NeurodataWithoutBorders/aqnwb/pull/224))
     * Added functionality to optionally create a simple example app that instantiates all generated classes to help test that all generated classes can be compiled (@oruebel, [#225](https://github.com/NeurodataWithoutBorders/aqnwb/pull/225))

docs/Doxyfile.in

@@ -26,7 +26,7 @@ EXPAND_ONLY_PREDEF   = YES
 # Add sources
 INPUT = "@PROJECT_SOURCE_DIR@/src" "@PROJECT_SOURCE_DIR@/docs/pages"
 RECURSIVE = YES
-EXAMPLE_PATH = "@PROJECT_SOURCE_DIR@/tests"  "@PROJECT_SOURCE_DIR@/.github/CODE_OF_CONDUCT.md"  "@PROJECT_SOURCE_DIR@/Legal.txt" "@PROJECT_SOURCE_DIR@/LICENSE"  "@PROJECT_SOURCE_DIR@/demo/labmetadata_extension_demo/src" "@PROJECT_SOURCE_DIR@/CHANGELOG.md"
+EXAMPLE_PATH = "@PROJECT_SOURCE_DIR@/tests"  "@PROJECT_SOURCE_DIR@/.github/CODE_OF_CONDUCT.md"  "@PROJECT_SOURCE_DIR@/Legal.txt" "@PROJECT_SOURCE_DIR@/LICENSE"  "@PROJECT_SOURCE_DIR@/demo/labmetadata_extension_demo/src" "@PROJECT_SOURCE_DIR@/CHANGELOG.md" "@PROJECT_SOURCE_DIR@/resources/utils"
 IMAGE_PATH = "@PROJECT_SOURCE_DIR@/resources/images"
 EXTRACT_ALL = YES
 RECURSIVE = YES
@@ -39,7 +39,7 @@ EXTRACT_STATIC      = YES
 # HIDE_UNDOC_MEMBERS = YES
 
 # Enable Markdown support
-MARKDOWN_SUPPORT = YES
+MARKDOWN_SUPPORT = YES 
 
 # Enable the call and caller graphs (this increases built time but seems reasonable for AqNWB)
 CALL_GRAPH           = YES

docs/pages/devdocs/install.dox

@@ -3,7 +3,9 @@
  *
  * \tableofcontents
  *
- * \section dev_requirements_sec Requirements
+ * \section dev_install_aqnwb_sec Installing AqNWB
+ *
+ * \subsection dev_requirements_sec Requirements
  *
  * Please ensure that the required libraries described in the
  * \ref user_requirements_sec "User Requirements" section are installed and
@@ -24,8 +26,7 @@
  *    - clang-format (optional, required for ``target=format-check``, ``target=format-fix``)
  *    - codespell  (optional, required for ``target=spell-check``, ``target=spell-fix``)
  *
- *
- * \section devbuild_sec Developer Build
+ * \subsection devbuild_sec Developer Build
  *
  * Build system targets that are only useful for developers of AqNWB are
  * hidden if the `aqnwb_DEVELOPER_MODE` option is disabled. Enabling this
@@ -58,7 +59,7 @@
  * The use of `HDF5_ROOT` and `BOOST_ROOT` environment variables is deprecated for modern CMake
  * and may not work reliably with recent CMake versions.
  *
- * \section  devbuild_presets_subsec Developer Presets
+ * \subsubsection  devbuild_presets_subsec Developer Presets
  *
  * As a developer, you can create your own dev preset by making a `CMakeUserPresets.json` file at the root of
  * the project:
@@ -103,7 +104,7 @@
  * Replace `<os>` in the `CMakeUserPresets.json` file with the name of
  * the operating system you have (`win64`, `linux` or `darwin`).
  *
- * \subsection configure_build_test Configure, Build and Test
+ * \subsubsection configure_build_test Configure, Build and Test
  *
  * You can configure, build and test the project respectively with the following commands from the project root on
  * any operating system with any build system:
@@ -114,18 +115,27 @@
  * ctest --preset=dev
  * \endcode
  *
- * \section devbuild_dev_mode_targets_subsec Developer Mode Targets
+ * \subsubsection devbuild_dev_mode_targets_subsec Developer Mode Targets
  *
  * Additional targets can be invoked when in development mode using the commands below
  *
  * \code{.sh}
  * cmake --build --preset=dev --target=<name of the target>
  * \endcode
  *
- * \subsection  devbuild_target_options_subsubsec Target options
+ * \paragraph  devbuild_target_options_subsubsec Target options
  * - `format-check`: run the `clang-format` tool on the codebase to check for formatting errors
  * - `format-fix` :  run the `clang-format` tool on the codebase with `FIX=YES` to both check and automatically fix for formatting errors
  * - `spell-check`: run the `codespell` tool on the codebase to check for common spelling errors
  * - `spell-fix` :  run the `codespell` tool on the codebase with `FIX=YES` to both check and automatically fix  common spelling errors
  * - `docs` : builds the documentation using Doxygen. (Note: run `cmake --preset=dev -DBUILD_DOCS=ON` before building to add docs target)
+ *
+ * \section dev_install_utils_sec Installing Python Utilities
+ *
+ * AqNWB provides a set of Python utilities for developers to help with generating C++ classes from NWB schema files.
+ * For details on how to use the utilities, see the [README.md](https://github.com/NeurodataWithoutBorders/aqnwb/blob/main/resources/utils/README.md)
+ * in the `resources/utils` directory.
+ *
+ * **README.md**
+ * \include resources/utils/README.md
  */

docs/pages/devdocs/integrating_extensions.dox

@@ -9,21 +9,24 @@
  * Integrating a new schema namespace with AqNWB (e.g., to support an extension to NWB) involves generating
  * the necessary specification files and ensuring that the namespace is registered with the 
  * \ref AQNWB::SPEC::NamespaceRegistry "NamespaceRegistry". This process is simplified through 
- * the use of the `generate_spec_files.py` script.
+ * the use of Python utilities that can be run with `uv`.
+ *
+ * @note
+ * See the \ref dev_install_utils_sec "Python Utilities" section in the developer installation guide for instructions on how to use the utilities with `uv`.
  *
  * 1. **Get the schema files**: Download or create the schema for the namespace in YAML format. 
  *    @note 
  *    If you are creating a new extension, please see the
  *    [NWB Extension Tutorial](https://nwb-overview.readthedocs.io/en/latest/extensions_tutorial/extensions_tutorial_home.html)
  *    for more information on how to create data schema for NWB. 
  * 
- * 2. **Convert the schema files to C++**: Run the `resources/utils/generate_spec_files.py` script on your
+ * 2. **Convert the schema files to C++**: Run the `generate-spec` command on your
  *    schema files to generate the necessary C++ header files. This script processes the schema and creates 
  *    the appropriate C++ header files that include the namespace definitions and registration. 
  *    @note
  *    To learn more about how to use the script and its parameters, you can view the help doc by running:
  *    @code 
- *    python resources/utils/generate_spec_files.py --help
+ *    uv run resources/utils/aqnwb_utils.py generate-spec --help
  *    @endcode
  *
  * 3. **Include the Generated Header Files**: In your C++ code that uses AqNWB, include the generated header files. The 
@@ -34,7 +37,7 @@
  *
  * 4. **Implement appropriate RegisteredType classes**: Follow the tutorial on
  *    \ref registered_type_page to define appropriate interfaces for the `neurodata_type`s defined in your new namespace. 
- *    \ref using_schematype_to_aqnwb can also provide additional help by providing a simple 
+ *    The `generate-types` command can also provide additional help by providing a simple 
  *    utility that can automatically generate skeleton AqNWB C++ classes for neurodata_types
  *    directly from JSON/YAML schema files.
  *
@@ -90,11 +93,11 @@
  *
  * @subsection labmetadata_extension_cpp_generation Step 2: Convert the Schema to C++
  *
- * The schema files are converted to C++ using the `resources/utils/generate_spec_files.py` script:
+ * The schema files are converted to C++ using the `generate-spec` command:
  *
  * @code
  * mkdir demo/labmetadata_extension_demo/src
- * python resources/utils/generate_spec_files.py demo/labmetadata_extension_demo/spec demo/labmetadata_extension_demo/src
+ * uv run resources/utils/aqnwb_utils.py generate-spec demo/labmetadata_extension_demo/spec demo/labmetadata_extension_demo/src
  * @endcode
  *
  * This generates the following new header file in the `demo/labmetadata_extension_demo/src` folder:

docs/pages/devdocs/nwb_schema.dox

@@ -19,7 +19,7 @@
  * 
  * This script will:
  *   - Clone the latest NWB and HDMF schema repositories into a temporary directory
- *   - Run `resources/utils/generate_spec_files.py` for both the NWB core and HDMF common schemas
+ *   - Run `uv run resources/utils/aqnwb_utils.py generate-spec` for both the NWB core and HDMF common schemas
  *   - Copy the generated C++ header files to `src/spec`
  *   - Clean up all temporary files automatically
  * 
@@ -33,16 +33,18 @@
  * PYTHON=python3 bash resources/utils/generate_nwb_schema_headers.sh
  * @endcode
  * 
- * For use with extensions and other advanced or custom use cases, developers may still run `resources/utils/generate_spec_files.py` directly.
+ * For use with extensions and other advanced or custom use cases, developers may still run the `generate-spec` command directly:
+ * @code{.sh}
+ * uv run resources/utils/aqnwb_utils.py generate-spec <schema_dir> <output_dir>
+ * @endcode
  *
  *
  * \section dev_docs_updating_nwb_schema_section Updating the schema
  *
  * Currently, the version of the schema being used for development is fixed and stored in the `/resources/schema` folder. 
  * Updating to a newer version of the schema requires:
- * - Regeneration of the `spec` header files via `resources/utils/generate_spec_files.py`
+ * - Regeneration of the `spec` header files via the `generate-spec` command
  * - Update of existing `Container` classes and unit tests in AqNWB to match changes in the  new schema compared to the previous schema
  * - Successful completion of all unit-test and round-trip testing with PyNWB and MatNWB
  *
  */
-

docs/pages/devdocs/registered_types.dox

@@ -415,9 +415,9 @@
  * in the \ref AQNWB::NWB::ElectrodesTable "ElectrodesTable" to read the `group_name` column
  * as `VectorData<std::string>` with the data type already specified as `std::string` at compile time.
  *
- * \section using_schematype_to_aqnwb Using the schematype_to_aqnwb.py Utility
+ * \section using_schematype_to_aqnwb Using the generate-types command
  *
- * The `resources/utils/schematype_to_aqnwb.py` script, included in the 
+ * The `generate-types` command, included in the 
  * [AqNWB source repository](https://github.com/NeurodataWithoutBorders/aqnwb),
  * is a simple utility designed to create skeleton C++ source files for integrating new 
  * neurodata_types with AqNWB. While the generated source files are only an outline and are not guaranteed to compile, 
@@ -427,20 +427,29 @@
  * for all neurodata_types in the NWB schema:
  * 
  * @code
- * python resources/utils/schematype_to_aqnwb.py nwb-schema/core/nwb.namespace.yaml test_output
+ * uv run resources/utils/aqnwb_utils.py generate-types nwb-schema/core/nwb.namespace.yaml test_output
  * @endcode
  *
  * The generated files will be placed in the folder hierarchy based on the name of the namespace and 
  * source yaml file where the type is defined. E.g, `TimeSeries` is defined in `nwb.base.yaml` in the 
  * `core` NWB namespace, and will be generated as `core/base/TimeSeries.hpp`. 
+ *
+ * We can also create a simple example app that instantiates all the generated classes to make it 
+ * simplify testing that all the classes can be compiled. To generate the app, simply add the
+ * `--generate-test-app` option: 
+ *
+ * @code
+ * uv run resources/utils/aqnwb_utils.py generate-types --generate-test-app nwb-schema/core/nwb.namespace.yaml test_output
+ * @endcode
+ *
  * To learn more about how to use the script and its parameters, you can view the help doc by running: 
  *
  * @code
- * python resources/utils/schematype_to_aqnwb.py --help
+ * uv run resources/utils/aqnwb_utils.py generate-types --help
  * @endcode
  *
  * \note
- * The `schematype_to_aqnwb.py` uses `PyNWB` for parsing schema. Currently the script does not unload
+ * The `generate-types` command uses `PyNWB` for parsing schema. Currently the script does not unload
  * namespaces loaded by default by `PyNWB`. I.e., if you see a warning of the form:
  * \code 
  * UserWarning: Ignoring cached namespace 'core' version 2.7.0 because version 2.8.0 is already loaded.
@@ -449,6 +458,10 @@
  * than the requested version. In practice, this is mainly relevant if you are generating classes for the
  * NWB `core` and `hdmf-common` namespaces.
  *
+ * \note
+ * When generating the test app via the `--generate-test-app` option we also
+ * need to generate the schema headers files and save them in the `/spec` folder.
+ *
  * \section implement_registered_type_unit_tests Testing RegisteredTypes
  *
  * As with all code, it is good practice to create appropriate unit tests to validate