From 7a3a68e55e84586becbb1fab1d8301bcce059340 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Mon, 29 Sep 2025 23:40:56 -0700 Subject: [PATCH 01/15] Add common Python ignores to .gitignore --- .gitignore | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitignore b/.gitignore index 82ce979eb..0b227cce9 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,12 @@ demo/cmake-build-*/ demo/inspect_electrical_series/*.nwb demo/*/Makefile demo/*/cmake_install.cmake + +# Python +__pycache__/ +*.pyc +*.pyo +*.pyd +.Python +env/ +venv/ From e320a9ee13c17f2466cd98295369d48aebd04a06 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 30 Sep 2025 00:16:45 -0700 Subject: [PATCH 02/15] Make the python utilities pip installable and add aqnwb-utils comman line utility --- .github/workflows/generate-classes.yml | 12 +-- .github/workflows/python-utils.yml | 10 +- .gitignore | 6 ++ docs/pages/2_devdocs.dox | 1 + docs/pages/devdocs/install.dox | 10 ++ docs/pages/devdocs/integrating_extensions.dox | 15 +-- docs/pages/devdocs/nwb_schema.dox | 7 +- docs/pages/devdocs/python_utils.dox | 98 +++++++++++++++++++ docs/pages/devdocs/registered_types.dox | 23 ++++- resources/utils/__init__.py | 1 + resources/utils/aqnwb_utils.py | 24 +++++ resources/utils/generate_spec_files.py | 5 +- resources/utils/setup.py | 16 +++ 13 files changed, 200 insertions(+), 28 deletions(-) create mode 100644 docs/pages/devdocs/python_utils.dox create mode 100644 resources/utils/__init__.py create mode 100644 resources/utils/aqnwb_utils.py create mode 100644 resources/utils/setup.py diff --git a/.github/workflows/generate-classes.yml b/.github/workflows/generate-classes.yml index be3d42714..a873d8693 100644 --- a/.github/workflows/generate-classes.yml +++ b/.github/workflows/generate-classes.yml @@ -43,7 +43,7 @@ jobs: - name: Install python script dependencies run: | python -m pip install --upgrade pip - pip install -r resources/utils/requirements.txt + pip install ./resources/utils # ----------------------------------------------------------------- # 2. LabMetadataExtension pipeline @@ -52,11 +52,11 @@ jobs: run: | mkdir test_output mkdir test_output/spec - python resources/utils/generate_spec_files.py demo/labmetadata_extension_demo/spec test_output/spec + aqnwb-utils generate-spec demo/labmetadata_extension_demo/spec test_output/spec - name: Run schematype_to_aqnwb.py to generate AqNWB classes for the LabMetadataExample extension run: | - python resources/utils/schematype_to_aqnwb.py --generate-test-app demo/labmetadata_extension_demo/spec/ndx-labmetadata-example.namespace.yaml test_output + aqnwb-utils generate-types --generate-test-app demo/labmetadata_extension_demo/spec/ndx-labmetadata-example.namespace.yaml test_output - name: List generated files for the LabMetadataExample extension run: | @@ -96,11 +96,11 @@ jobs: - name: Run generate_nwb_schema_headers.sh to generated headers the nwb_schema run: | mkdir test_output_nwb_schema/spec - python resources/utils/generate_spec_files.py test_output_nwb_schema/nwb-schema/core test_output_nwb_schema/spec + aqnwb-utils generate-spec test_output_nwb_schema/nwb-schema/core test_output_nwb_schema/spec - name: Run schematype_to_aqnwb.py to generate AqNWB classes for the nwb_schema run: | - python resources/utils/schematype_to_aqnwb.py --generate-test-app test_output_nwb_schema/nwb-schema/core/nwb.namespace.yaml test_output_nwb_schema + aqnwb-utils generate-types --generate-test-app test_output_nwb_schema/nwb-schema/core/nwb.namespace.yaml test_output_nwb_schema - name: List generated files for the nwb-schema run: | @@ -118,5 +118,3 @@ jobs: run: | cd test_output_nwb_schema/test_app/build ./bin/schema_compilation_test - - \ No newline at end of file diff --git a/.github/workflows/python-utils.yml b/.github/workflows/python-utils.yml index 2f2bfaffb..0adcec7dd 100644 --- a/.github/workflows/python-utils.yml +++ b/.github/workflows/python-utils.yml @@ -22,7 +22,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r resources/utils/requirements.txt + pip install ./resources/utils - name: Clone latest NWB schema release run: | @@ -37,17 +37,17 @@ jobs: echo "Checking out NWB schema at tag: $NWB_LATEST_TAG" git checkout "$NWB_LATEST_TAG" - - name: Run generate_spec_files.py + - name: Run generate_spec_files run: | - output=$(python resources/utils/generate_spec_files.py test_output/nwb-schema/core test_output 2>&1) + output=$(aqnwb-utils generate-spec test_output/nwb-schema/core test_output 2>&1) echo "$output" if echo "$output" | grep -q "ERROR"; then exit 1 fi - - name: Run schematype_to_aqnwb.py + - name: Run schematype_to_aqnwb run: | - output=$(python resources/utils/schematype_to_aqnwb.py test_output/nwb-schema/core/nwb.namespace.yaml test_output 2>&1) + output=$(aqnwb-utils generate-types test_output/nwb-schema/core/nwb.namespace.yaml test_output 2>&1) echo "$output" if echo "$output" | grep -q "ERROR"; then exit 1 diff --git a/.gitignore b/.gitignore index 0b227cce9..9584fc7a4 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,9 @@ __pycache__/ .Python env/ venv/ + +# Python packaging +build/ +dist/ +*.egg-info/ +wheels/ diff --git a/docs/pages/2_devdocs.dox b/docs/pages/2_devdocs.dox index 9d2344424..cd6d5fa2b 100644 --- a/docs/pages/2_devdocs.dox +++ b/docs/pages/2_devdocs.dox @@ -4,6 +4,7 @@ * This documentation is intended for developers of AqNWB. * * - \subpage dev_install_page + * - \subpage python_utils_page * - \subpage testing * - \subpage dev_docs_page * - \subpage nwb_schema_page diff --git a/docs/pages/devdocs/install.dox b/docs/pages/devdocs/install.dox index 0965bd0fd..3118bad00 100644 --- a/docs/pages/devdocs/install.dox +++ b/docs/pages/devdocs/install.dox @@ -24,6 +24,16 @@ * - clang-format (optional, required for ``target=format-check``, ``target=format-fix``) * - codespell (optional, required for ``target=spell-check``, ``target=spell-fix``) * + * \section dev_utils_sec Python Utilities + * + * AqNWB provides a set of Python utilities for developers to help with generating C++ classes from NWB schema files. + * To install these utilities, run the following command from the root of the project: + * + * \code{.sh} + * pip install ./resources/utils + * \endcode + * + * This will install the `aqnwb-utils` command-line utility, which can be used to access the various Python utilities. * * \section devbuild_sec Developer Build * diff --git a/docs/pages/devdocs/integrating_extensions.dox b/docs/pages/devdocs/integrating_extensions.dox index bf7592335..4a84639b4 100644 --- a/docs/pages/devdocs/integrating_extensions.dox +++ b/docs/pages/devdocs/integrating_extensions.dox @@ -9,7 +9,10 @@ * Integrating a new schema namespace with AqNWB (e.g., to support an extension to NWB) involves generating * the necessary specification files and ensuring that the namespace is registered with the * \ref AQNWB::SPEC::NamespaceRegistry "NamespaceRegistry". This process is simplified through - * the use of the `generate_spec_files.py` script. + * the use of the `aqnwb-utils` command-line utility. + * + * @note + * See the \ref dev_utils_sec "Python Utilities" section in the developer installation guide for instructions on how to install the utilities. * * 1. **Get the schema files**: Download or create the schema for the namespace in YAML format. * @note @@ -17,13 +20,13 @@ * [NWB Extension Tutorial](https://nwb-overview.readthedocs.io/en/latest/extensions_tutorial/extensions_tutorial_home.html) * for more information on how to create data schema for NWB. * - * 2. **Convert the schema files to C++**: Run the `resources/utils/generate_spec_files.py` script on your + * 2. **Convert the schema files to C++**: Run the `aqnwb-utils generate-spec` command on your * schema files to generate the necessary C++ header files. This script processes the schema and creates * the appropriate C++ header files that include the namespace definitions and registration. * @note * To learn more about how to use the script and its parameters, you can view the help doc by running: * @code - * python resources/utils/generate_spec_files.py --help + * aqnwb-utils generate-spec --help * @endcode * * 3. **Include the Generated Header Files**: In your C++ code that uses AqNWB, include the generated header files. The @@ -34,7 +37,7 @@ * * 4. **Implement appropriate RegisteredType classes**: Follow the tutorial on * \ref registered_type_page to define appropriate interfaces for the `neurodata_type`s defined in your new namespace. - * \ref using_schematype_to_aqnwb can also provide additional help by providing a simple + * The `aqnwb-utils generate-types` command can also provide additional help by providing a simple * utility that can automatically generate skeleton AqNWB C++ classes for neurodata_types * directly from JSON/YAML schema files. * @@ -90,11 +93,11 @@ * * @subsection labmetadata_extension_cpp_generation Step 2: Convert the Schema to C++ * - * The schema files are converted to C++ using the `resources/utils/generate_spec_files.py` script: + * The schema files are converted to C++ using the `aqnwb-utils generate-spec` command: * * @code * mkdir demo/labmetadata_extension_demo/src - * python resources/utils/generate_spec_files.py demo/labmetadata_extension_demo/spec demo/labmetadata_extension_demo/src + * aqnwb-utils generate-spec demo/labmetadata_extension_demo/spec demo/labmetadata_extension_demo/src * @endcode * * This generates the following new header file in the `demo/labmetadata_extension_demo/src` folder: diff --git a/docs/pages/devdocs/nwb_schema.dox b/docs/pages/devdocs/nwb_schema.dox index 7a4d6ece8..4e0220e30 100644 --- a/docs/pages/devdocs/nwb_schema.dox +++ b/docs/pages/devdocs/nwb_schema.dox @@ -19,7 +19,7 @@ * * This script will: * - Clone the latest NWB and HDMF schema repositories into a temporary directory - * - Run `resources/utils/generate_spec_files.py` for both the NWB core and HDMF common schemas + * - Run `aqnwb-utils generate-spec` for both the NWB core and HDMF common schemas * - Copy the generated C++ header files to `src/spec` * - Clean up all temporary files automatically * @@ -33,16 +33,15 @@ * PYTHON=python3 bash resources/utils/generate_nwb_schema_headers.sh * @endcode * - * For use with extensions and other advanced or custom use cases, developers may still run `resources/utils/generate_spec_files.py` directly. + * For use with extensions and other advanced or custom use cases, developers may still run `aqnwb-utils generate-spec` directly. * * * \section dev_docs_updating_nwb_schema_section Updating the schema * * Currently, the version of the schema being used for development is fixed and stored in the `/resources/schema` folder. * Updating to a newer version of the schema requires: - * - Regeneration of the `spec` header files via `resources/utils/generate_spec_files.py` + * - Regeneration of the `spec` header files via `aqnwb-utils generate-spec` * - Update of existing `Container` classes and unit tests in AqNWB to match changes in the new schema compared to the previous schema * - Successful completion of all unit-test and round-trip testing with PyNWB and MatNWB * */ - diff --git a/docs/pages/devdocs/python_utils.dox b/docs/pages/devdocs/python_utils.dox new file mode 100644 index 000000000..81bac8877 --- /dev/null +++ b/docs/pages/devdocs/python_utils.dox @@ -0,0 +1,98 @@ +/** + * \page python_utils_page Python Utilities 🐍 + * + * \tableofcontents + * + * AqNWB provides a set of Python utilities for developers to help with generating C++ classes from NWB schema files. + * These utilities are available as a command-line tool called `aqnwb-utils`. + * + * \section install_utils_sec Installing the Utilities + * + * To install the Python utilities, run the following command from the root of the project: + * + * \code{.sh} + * pip install ./resources/utils + * \endcode + * + * This will install the `aqnwb-utils` command-line utility, which can be used to access the various Python utilities. + * + * \section using_utils_sec Using the Utilities + * + * The `aqnwb-utils` command-line utility provides two main commands: + * + * - `generate-spec`: This command generates C++ header files from NWB schema files. + * - `generate-types`: This command generates skeleton C++ source files for integrating new neurodata_types with AqNWB. + * + * You can get more information about each command by running it with the `--help` flag. + * + * \subsection generate_spec_sec generate-spec + * + * The `generate-spec` command is used to generate C++ header files from NWB schema files. + * + * \code{.sh} + * aqnwb-utils generate-spec + * \endcode + * + * - ``: The directory containing the NWB schema files. + * - ``: The directory where the generated C++ header files will be saved. + * + * \subsection generate_types_sec generate-types + * + * The `generate-types` command is used to generate skeleton C++ source files for integrating new neurodata_types with AqNWB. + * + * \code{.sh} + * aqnwb-utils generate-types + * \endcode + * + * - ``: The path to the namespace file (e.g., `nwb.namespace.yaml`). + * - ``: The directory where the generated C++ source files will be saved. + * + * \section create_utils_sec Creating a New Utility + * + * To create a new utility, you need to: + * + * 1. Create a new Python file in the `resources/utils` directory. + * 2. In the new file, define a function that takes `argparse.ArgumentParser` as input and adds the necessary arguments for your utility. + * 3. In the same file, define a function that takes the parsed arguments as input and implements the logic for your utility. + * 4. In `resources/utils/aqnwb_utils.py`, import your new functions and add a new subcommand to the `subparsers` object. + * + * For example, to add a new command called `my-command`, you would: + * + * 1. Create a new file `resources/utils/my_command.py` with the following content: + * + * \code{.py} + * def setup_parser(parser): + * parser.add_argument("my_arg", help="My argument") + * + * def main(args): + * print(f"My argument is {args.my_arg}") + * + * if __name__ == "__main__": + * import argparse + * parser = argparse.ArgumentParser() + * setup_parser(parser) + * args = parser.parse_args() + * main(args) + * \endcode + * + * 2. In `resources/utils/aqnwb_utils.py`, add the following: + * + * \code{.py} + * from . import my_command + * + * # ... + * + * def main(): + * # ... + * subparsers = parser.add_subparsers(dest="command") + * # ... + * my_command_parser = subparsers.add_parser("my-command", help="My command") + * my_command.setup_parser(my_command_parser) + * + * # ... + * + * if args.command == "my-command": + * my_command.main(args) + * \endcode + * + */ diff --git a/docs/pages/devdocs/registered_types.dox b/docs/pages/devdocs/registered_types.dox index dedf172ce..ecd484303 100644 --- a/docs/pages/devdocs/registered_types.dox +++ b/docs/pages/devdocs/registered_types.dox @@ -415,9 +415,9 @@ * in the \ref AQNWB::NWB::ElectrodesTable "ElectrodesTable" to read the `group_name` column * as `VectorData` with the data type already specified as `std::string` at compile time. * - * \section using_schematype_to_aqnwb Using the schematype_to_aqnwb.py Utility + * \section using_schematype_to_aqnwb Using the aqnwb-utils generate-types command * - * The `resources/utils/schematype_to_aqnwb.py` script, included in the + * The `aqnwb-utils generate-types` command, included in the * [AqNWB source repository](https://github.com/NeurodataWithoutBorders/aqnwb), * is a simple utility designed to create skeleton C++ source files for integrating new * neurodata_types with AqNWB. While the generated source files are only an outline and are not guaranteed to compile, @@ -427,20 +427,29 @@ * for all neurodata_types in the NWB schema: * * @code - * python resources/utils/schematype_to_aqnwb.py nwb-schema/core/nwb.namespace.yaml test_output + * aqnwb-utils generate-types nwb-schema/core/nwb.namespace.yaml test_output * @endcode * * The generated files will be placed in the folder hierarchy based on the name of the namespace and * source yaml file where the type is defined. E.g, `TimeSeries` is defined in `nwb.base.yaml` in the * `core` NWB namespace, and will be generated as `core/base/TimeSeries.hpp`. + * + * We can also create a simple example app that instantiates all the generated classes to make it + * simplify testing that all the classes can be compiled. To generate the app, simply add the + * `--generate-test-app` option: + * + * @code + * aqnwb-utils generate-types --generate-test-app nwb-schema/core/nwb.namespace.yaml test_output + * @endcode + * * To learn more about how to use the script and its parameters, you can view the help doc by running: * * @code - * python resources/utils/schematype_to_aqnwb.py --help + * aqnwb-utils generate-types --help * @endcode * * \note - * The `schematype_to_aqnwb.py` uses `PyNWB` for parsing schema. Currently the script does not unload + * The `aqnwb-utils generate-types` command uses `PyNWB` for parsing schema. Currently the script does not unload * namespaces loaded by default by `PyNWB`. I.e., if you see a warning of the form: * \code * UserWarning: Ignoring cached namespace 'core' version 2.7.0 because version 2.8.0 is already loaded. @@ -449,6 +458,10 @@ * than the requested version. In practice, this is mainly relevant if you are generating classes for the * NWB `core` and `hdmf-common` namespaces. * + * \note + * When generating the test app via `aqnwb-utils generate-types --generate-test-app` we also + * need to generate the schema headers files and save them in the `/spec` folder. + * * \section implement_registered_type_unit_tests Testing RegisteredTypes * * As with all code, it is good practice to create appropriate unit tests to validate diff --git a/resources/utils/__init__.py b/resources/utils/__init__.py new file mode 100644 index 000000000..c441c4812 --- /dev/null +++ b/resources/utils/__init__.py @@ -0,0 +1 @@ +# This file makes the utils directory a package diff --git a/resources/utils/aqnwb_utils.py b/resources/utils/aqnwb_utils.py new file mode 100644 index 000000000..12ef6800f --- /dev/null +++ b/resources/utils/aqnwb_utils.py @@ -0,0 +1,24 @@ +import argparse +import generate_spec_files +import schematype_to_aqnwb + +def main(): + parser = argparse.ArgumentParser(description="AQNWB utilities") + subparsers = parser.add_subparsers(dest="command") + + # Sub-parser for generating spec files + parser_spec = subparsers.add_parser("generate-spec", help="Generate spec files") + parser_spec.set_defaults(func=generate_spec_files.main) + + # Sub-parser for generating types + parser_types = subparsers.add_parser("generate-types", help="Generate neurodata types") + parser_types.set_defaults(func=schematype_to_aqnwb.main) + + args = parser.parse_args() + if hasattr(args, 'func'): + args.func() + else: + parser.print_help() + +if __name__ == "__main__": + main() diff --git a/resources/utils/generate_spec_files.py b/resources/utils/generate_spec_files.py index 15933b180..2c85845a4 100644 --- a/resources/utils/generate_spec_files.py +++ b/resources/utils/generate_spec_files.py @@ -187,7 +187,7 @@ def process_schema_files(schema_dir: Path, output_dir: Path, chunk_size: int) -> process_namespace_file(file, output_dir, chunk_size) logger.info(f"Finished processing schema files in directory: {schema_dir}") -if __name__ == '__main__': +def main(): parser = argparse.ArgumentParser(description='Process schema files.') parser.add_argument('schema_dir', type=Path, nargs='?', default=Path('./resources/schema/'), help='Directory containing the schema files') parser.add_argument('output_dir', type=Path, nargs='?', default=Path('./src/spec/'), help='Directory to output the generated header files') @@ -195,3 +195,6 @@ def process_schema_files(schema_dir: Path, output_dir: Path, chunk_size: int) -> args = parser.parse_args() process_schema_files(args.schema_dir, args.output_dir, args.chunk_size) + +if __name__ == '__main__': + main() diff --git a/resources/utils/setup.py b/resources/utils/setup.py new file mode 100644 index 000000000..80b9988d9 --- /dev/null +++ b/resources/utils/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup, find_packages + +with open("requirements.txt") as f: + install_requires = f.read().strip().split("\n") + +setup( + name="aqnwb-utils", + version="0.1.0", + packages=find_packages(), + install_requires=install_requires, + entry_points={ + "console_scripts": [ + "aqnwb-utils=aqnwb_utils:main", + ] + }, +) From 0ee5ac30b55c0ddc31d536656dd012f7bbc5fb12 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 30 Sep 2025 00:30:56 -0700 Subject: [PATCH 03/15] Update CHANGELOG --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cabd142e3..962d86d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +* Added `aqnwb-utils` as a pip-installable command-line utility to provide a common interface for aqnwb command line tools, e.g., `schematype_to_aqnwb.py` and `generate_spec_files.py` (@oruebel, [#227](https://github.com/NeurodataWithoutBorders/aqnwb/pull/227)) + + ### Changed * Enhanced the `schematype_to_aqnwb` utility script: * Generated source files are now placed into a folder hierarchy based on the name of the namespace and schemafile of the neurodata_type (@oruebel, [#224](https://github.com/NeurodataWithoutBorders/aqnwb/pull/224)) * Added functionality to optionally create a simple example app that instantiates all generated classes to help test that all generated classes can be compiled (@oruebel, [#225](https://github.com/NeurodataWithoutBorders/aqnwb/pull/225)) * Updated generation of header files to ensure proper compilation, e.g.: i) identify and include the headers of all neurodata_types that are being used, ii) fix formatting of comments to avoid nested multi-line comments, iii) fixed issues with incomplete typenames (@oruebel, [#225](https://github.com/NeurodataWithoutBorders/aqnwb/pull/225)) * Added GitHub action testing that all sources files generated by the `schematype_to_aqnwb` utility for the nwb-schema and LabMetaDataExtension example can be compiled (@oruebel, [#225](https://github.com/NeurodataWithoutBorders/aqnwb/pull/225)) +* Updated documentation to refer to the new `aqnwb-utils` command-line utility (@oruebel, [#226](https://github.com/NeurodataWithoutBorders/aqnwb/pull/226)) From 562294ce275d60436d5b3ed22f79ef1e8ef212a1 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 30 Sep 2025 00:31:21 -0700 Subject: [PATCH 04/15] Fix argument parsing for command line utilities --- resources/utils/aqnwb_utils.py | 4 ++- resources/utils/generate_spec_files.py | 10 ++++--- resources/utils/schematype_to_aqnwb.py | 36 +++++++++++++++----------- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/resources/utils/aqnwb_utils.py b/resources/utils/aqnwb_utils.py index 12ef6800f..39d1328a1 100644 --- a/resources/utils/aqnwb_utils.py +++ b/resources/utils/aqnwb_utils.py @@ -8,15 +8,17 @@ def main(): # Sub-parser for generating spec files parser_spec = subparsers.add_parser("generate-spec", help="Generate spec files") + generate_spec_files.setup_parser(parser_spec) parser_spec.set_defaults(func=generate_spec_files.main) # Sub-parser for generating types parser_types = subparsers.add_parser("generate-types", help="Generate neurodata types") + schematype_to_aqnwb.setup_parser(parser_types) parser_types.set_defaults(func=schematype_to_aqnwb.main) args = parser.parse_args() if hasattr(args, 'func'): - args.func() + args.func(args) else: parser.print_help() diff --git a/resources/utils/generate_spec_files.py b/resources/utils/generate_spec_files.py index 2c85845a4..180fa8621 100644 --- a/resources/utils/generate_spec_files.py +++ b/resources/utils/generate_spec_files.py @@ -187,12 +187,16 @@ def process_schema_files(schema_dir: Path, output_dir: Path, chunk_size: int) -> process_namespace_file(file, output_dir, chunk_size) logger.info(f"Finished processing schema files in directory: {schema_dir}") -def main(): - parser = argparse.ArgumentParser(description='Process schema files.') +def setup_parser(parser): parser.add_argument('schema_dir', type=Path, nargs='?', default=Path('./resources/schema/'), help='Directory containing the schema files') parser.add_argument('output_dir', type=Path, nargs='?', default=Path('./src/spec/'), help='Directory to output the generated header files') parser.add_argument('--chunk-size', type=int, default=16000, help='Size of the chunks for splitting large JSON strings') - args = parser.parse_args() + +def main(args=None): + if args is None: + parser = argparse.ArgumentParser(description='Process schema files.') + setup_parser(parser) + args = parser.parse_args() process_schema_files(args.schema_dir, args.output_dir, args.chunk_size) diff --git a/resources/utils/schematype_to_aqnwb.py b/resources/utils/schematype_to_aqnwb.py index 3e548d63e..4ca9e09e5 100755 --- a/resources/utils/schematype_to_aqnwb.py +++ b/resources/utils/schematype_to_aqnwb.py @@ -1307,7 +1307,21 @@ def generate_test_app( logger.error(f"Failed to generate test application: {e}") -def main() -> None: +def setup_parser(parser): + """ + Set up argument parser for the script. + """ + parser.add_argument( + "schema_file", help="Path to the namespace schema file (JSON or YAML)" + ) + parser.add_argument("output_dir", help="Directory to output the generated code") + parser.add_argument( + "--generate-test-app", + action="store_true", + help="Generate a test application to verify compilation of all generated classes" + ) + +def main(args=None) -> None: """ Main function to parse arguments and generate code. @@ -1320,20 +1334,12 @@ def main() -> None: Returns: None """ - parser = argparse.ArgumentParser( - description="Generate C++ code from NWB schema files." - ) - parser.add_argument( - "schema_file", help="Path to the namespace schema file (JSON or YAML)" - ) - parser.add_argument("output_dir", help="Directory to output the generated code") - parser.add_argument( - "--generate-test-app", - action="store_true", - help="Generate a test application to verify compilation of all generated classes" - ) - - args = parser.parse_args() + if args is None: + parser = argparse.ArgumentParser( + description="Generate C++ code from NWB schema files." + ) + setup_parser(parser) + args = parser.parse_args() try: logger.info(f"Parsing schema file: {args.schema_file}") From e97db8816e94ab96814eb407842bb1f25f996fe4 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 30 Sep 2025 00:34:55 -0700 Subject: [PATCH 05/15] Fix argument parsing for command line utilities --- resources/utils/generate_spec_files.py | 12 +++++------- resources/utils/schematype_to_aqnwb.py | 16 +++++++--------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/resources/utils/generate_spec_files.py b/resources/utils/generate_spec_files.py index 180fa8621..527baae21 100644 --- a/resources/utils/generate_spec_files.py +++ b/resources/utils/generate_spec_files.py @@ -192,13 +192,11 @@ def setup_parser(parser): parser.add_argument('output_dir', type=Path, nargs='?', default=Path('./src/spec/'), help='Directory to output the generated header files') parser.add_argument('--chunk-size', type=int, default=16000, help='Size of the chunks for splitting large JSON strings') -def main(args=None): - if args is None: - parser = argparse.ArgumentParser(description='Process schema files.') - setup_parser(parser) - args = parser.parse_args() - +def main(args): process_schema_files(args.schema_dir, args.output_dir, args.chunk_size) if __name__ == '__main__': - main() + parser = argparse.ArgumentParser(description='Process schema files.') + setup_parser(parser) + args = parser.parse_args() + main(args) diff --git a/resources/utils/schematype_to_aqnwb.py b/resources/utils/schematype_to_aqnwb.py index 4ca9e09e5..3c2e75731 100755 --- a/resources/utils/schematype_to_aqnwb.py +++ b/resources/utils/schematype_to_aqnwb.py @@ -1321,7 +1321,7 @@ def setup_parser(parser): help="Generate a test application to verify compilation of all generated classes" ) -def main(args=None) -> None: +def main(args) -> None: """ Main function to parse arguments and generate code. @@ -1334,13 +1334,6 @@ def main(args=None) -> None: Returns: None """ - if args is None: - parser = argparse.ArgumentParser( - description="Generate C++ code from NWB schema files." - ) - setup_parser(parser) - args = parser.parse_args() - try: logger.info(f"Parsing schema file: {args.schema_file}") namespace, neurodata_types, type_to_file_map, type_to_namespace_map = parse_schema_file(Path(args.schema_file)) @@ -1437,4 +1430,9 @@ def main(args=None) -> None: if __name__ == "__main__": - main() + parser = argparse.ArgumentParser( + description="Generate C++ code from NWB schema files." + ) + setup_parser(parser) + args = parser.parse_args() + main(args) From ce417f2f2eabe7624f668f1c9679228f0ed98925 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 30 Sep 2025 00:37:51 -0700 Subject: [PATCH 06/15] Add github workflow to debug issue --- .github/workflows/debug-python-utils.yml | 59 ++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/debug-python-utils.yml diff --git a/.github/workflows/debug-python-utils.yml b/.github/workflows/debug-python-utils.yml new file mode 100644 index 000000000..31ffa6152 --- /dev/null +++ b/.github/workflows/debug-python-utils.yml @@ -0,0 +1,59 @@ +name: Debug Utility Scripts + +on: + push: + branches: + - main + pull_request: + +jobs: + debug-run-scripts: + runs-on: macos-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ./resources/utils + + - name: Clone latest NWB schema release + run: | + mkdir -p test_output + git clone https://github.com/NeurodataWithoutBorders/nwb-schema.git test_output/nwb-schema + cd test_output/nwb-schema + NWB_LATEST_TAG=$(git tag --sort=-v:refname | head -n 1) + if [ -z "$NWB_LATEST_TAG" ]; then + echo "ERROR: Could not determine latest NWB schema release tag." + exit 1 + fi + echo "Checking out NWB schema at tag: $NWB_LATEST_TAG" + git checkout "$NWB_LATEST_TAG" + cd ../.. + echo "Listing contents of test_output/nwb-schema/core:" + ls -la test_output/nwb-schema/core + + - name: Run generate_spec_files with debug output + run: | + echo "Running aqnwb-utils generate-spec..." + aqnwb-utils generate-spec test_output/nwb-schema/core test_output + echo "Finished running aqnwb-utils generate-spec." + echo "Listing contents of test_output:" + ls -la test_output + echo "Contents of test_output/core.hpp:" + cat test_output/core.hpp || echo "core.hpp not found" + + - name: Run schematype_to_aqnwb with debug output + run: | + echo "Running aqnwb-utils generate-types..." + aqnwb-utils generate-types test_output/nwb-schema/core/nwb.namespace.yaml test_output + echo "Finished running aqnwb-utils generate-types." + echo "Listing contents of test_output:" + ls -laR test_output From 88e1c58d1ff4fd3b22686ccd4d411742e528b439 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 30 Sep 2025 00:40:30 -0700 Subject: [PATCH 07/15] Fix setup for command utils --- resources/utils/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/utils/setup.py b/resources/utils/setup.py index 80b9988d9..17e87e5f8 100644 --- a/resources/utils/setup.py +++ b/resources/utils/setup.py @@ -7,6 +7,7 @@ name="aqnwb-utils", version="0.1.0", packages=find_packages(), + py_modules=["aqnwb_utils", "generate_spec_files", "schematype_to_aqnwb"], install_requires=install_requires, entry_points={ "console_scripts": [ From b2f956fe8e61772653354c0c68b4cb2ae7300af2 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 30 Sep 2025 00:41:58 -0700 Subject: [PATCH 08/15] Removed debug workflow --- .github/workflows/debug-python-utils.yml | 59 ------------------------ 1 file changed, 59 deletions(-) delete mode 100644 .github/workflows/debug-python-utils.yml diff --git a/.github/workflows/debug-python-utils.yml b/.github/workflows/debug-python-utils.yml deleted file mode 100644 index 31ffa6152..000000000 --- a/.github/workflows/debug-python-utils.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Debug Utility Scripts - -on: - push: - branches: - - main - pull_request: - -jobs: - debug-run-scripts: - runs-on: macos-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install ./resources/utils - - - name: Clone latest NWB schema release - run: | - mkdir -p test_output - git clone https://github.com/NeurodataWithoutBorders/nwb-schema.git test_output/nwb-schema - cd test_output/nwb-schema - NWB_LATEST_TAG=$(git tag --sort=-v:refname | head -n 1) - if [ -z "$NWB_LATEST_TAG" ]; then - echo "ERROR: Could not determine latest NWB schema release tag." - exit 1 - fi - echo "Checking out NWB schema at tag: $NWB_LATEST_TAG" - git checkout "$NWB_LATEST_TAG" - cd ../.. - echo "Listing contents of test_output/nwb-schema/core:" - ls -la test_output/nwb-schema/core - - - name: Run generate_spec_files with debug output - run: | - echo "Running aqnwb-utils generate-spec..." - aqnwb-utils generate-spec test_output/nwb-schema/core test_output - echo "Finished running aqnwb-utils generate-spec." - echo "Listing contents of test_output:" - ls -la test_output - echo "Contents of test_output/core.hpp:" - cat test_output/core.hpp || echo "core.hpp not found" - - - name: Run schematype_to_aqnwb with debug output - run: | - echo "Running aqnwb-utils generate-types..." - aqnwb-utils generate-types test_output/nwb-schema/core/nwb.namespace.yaml test_output - echo "Finished running aqnwb-utils generate-types." - echo "Listing contents of test_output:" - ls -laR test_output From ea3860c51873ae7a3acc30735c957e69225e55d6 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 30 Sep 2025 00:53:53 -0700 Subject: [PATCH 09/15] Update example app generation to work with installed command line utilities --- resources/utils/schematype_to_aqnwb.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/resources/utils/schematype_to_aqnwb.py b/resources/utils/schematype_to_aqnwb.py index 3c2e75731..226e50ee5 100755 --- a/resources/utils/schematype_to_aqnwb.py +++ b/resources/utils/schematype_to_aqnwb.py @@ -1013,7 +1013,7 @@ def generate_test_app_cmake(output_dir: Path, app_name: str, cpp_files: List[str # Calculate the path to AqNWB source directory relative to script location # Script is in resources/utils, so AqNWB src is ../../src relative to script aqnwb_src_dir = script_dir.parent.parent / "src" - aqnwb_build_dir = script_dir.parent.parent / "build" / "dev" + aqnwb_build_dir = script_dir.parent.parent / "build" aqnwb_libs_dir = script_dir.parent.parent / "libs" cmake_content = f"""cmake_minimum_required(VERSION 3.15) @@ -1057,21 +1057,22 @@ def generate_test_app_cmake(output_dir: Path, app_name: str, cpp_files: List[str target_include_directories({app_name} PRIVATE ${{AQNWB_SRC_DIR}} ${{CMAKE_CURRENT_SOURCE_DIR}}/.. + ${{CMAKE_CURRENT_SOURCE_DIR}}/../spec ${{HDF5_INCLUDE_DIRS}} ${{Boost_INCLUDE_DIRS}} ) # Find the aqnwb library -if(EXISTS "${{AQNWB_DIR}}/libaqnwb.a") - set(AQNWB_LIBRARY "${{AQNWB_DIR}}/libaqnwb.a") -elseif(EXISTS "${{AQNWB_DIR}}/libaqnwb.so") - set(AQNWB_LIBRARY "${{AQNWB_DIR}}/libaqnwb.so") -elseif(EXISTS "${{AQNWB_DIR}}/libaqnwb.dylib") - set(AQNWB_LIBRARY "${{AQNWB_DIR}}/libaqnwb.dylib") -else() +find_library(AQNWB_LIBRARY + NAMES aqnwb + HINTS "${{AQNWB_DIR}}" + PATH_SUFFIXES "lib" "bin" +) +if (NOT AQNWB_LIBRARY) message(FATAL_ERROR "Could not find aqnwb library in ${{AQNWB_DIR}}. Please build the main project first or set AQNWB_DIR to the correct path.") endif() + # Link libraries target_link_libraries({app_name} ${{AQNWB_LIBRARY}} From 5ba6b982a804010524c6ec5e08ff8522c063aaef Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 30 Sep 2025 01:02:14 -0700 Subject: [PATCH 10/15] Update example app generation to work with installed command line utilities --- .github/workflows/generate-classes.yml | 5 ++- .gitignore | 7 ++++ resources/utils/schematype_to_aqnwb.py | 45 ++++---------------------- 3 files changed, 15 insertions(+), 42 deletions(-) diff --git a/.github/workflows/generate-classes.yml b/.github/workflows/generate-classes.yml index a873d8693..1e7d90a1e 100644 --- a/.github/workflows/generate-classes.yml +++ b/.github/workflows/generate-classes.yml @@ -67,9 +67,8 @@ jobs: cd test_output/test_app mkdir build cd build - cmake -DAQNWB_DIR="${{ github.workspace }}/build" ../ + cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" ../ make -j 2 - ./bin/schema_compilation_test - name: Run the test program for LabMetadataExample extension run: | @@ -111,7 +110,7 @@ jobs: cd test_output_nwb_schema/test_app mkdir build cd build - cmake -DAQNWB_DIR="${{ github.workspace }}/build" ../ + cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" ../ make -j 2 - name: Run the test program for the nwb_schema diff --git a/.gitignore b/.gitignore index 9584fc7a4..1bef69b97 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,10 @@ build/ dist/ *.egg-info/ wheels/ + +# Jupyter Notebook +.ipynb_checkpoints + +# Pytest +.pytest_cache/ +htmlcov/ diff --git a/resources/utils/schematype_to_aqnwb.py b/resources/utils/schematype_to_aqnwb.py index 226e50ee5..780052927 100755 --- a/resources/utils/schematype_to_aqnwb.py +++ b/resources/utils/schematype_to_aqnwb.py @@ -1010,12 +1010,6 @@ def generate_test_app_cmake(output_dir: Path, app_name: str, cpp_files: List[str # Convert file paths to use forward slashes for CMake cpp_files_cmake = [f.replace("\\", "/") for f in cpp_files] - # Calculate the path to AqNWB source directory relative to script location - # Script is in resources/utils, so AqNWB src is ../../src relative to script - aqnwb_src_dir = script_dir.parent.parent / "src" - aqnwb_build_dir = script_dir.parent.parent / "build" - aqnwb_libs_dir = script_dir.parent.parent / "libs" - cmake_content = f"""cmake_minimum_required(VERSION 3.15) project({app_name} VERSION 0.1.0 LANGUAGES CXX) @@ -1023,20 +1017,9 @@ def generate_test_app_cmake(output_dir: Path, app_name: str, cpp_files: List[str set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -# Allow configuring paths to dependencies based on script location -set(AQNWB_SRC_DIR "{aqnwb_src_dir.as_posix()}" CACHE PATH "Path to aqnwb source directory") -set(AQNWB_DIR "{aqnwb_build_dir.as_posix()}" CACHE PATH "Path to aqnwb build directory") -set(HDF5_DIR "{(aqnwb_libs_dir / "hdf5_build" / "install" / "cmake").as_posix()}" CACHE PATH "Path to HDF5 build directory") -set(BOOST_ROOT "{(aqnwb_libs_dir / "boost_install").as_posix()}" CACHE PATH "Path to Boost root directory") - -# Disable compiler flags that cause issues on macOS -if(APPLE) - set(CMAKE_CXX_FLAGS "${{CMAKE_CXX_FLAGS}} -Wno-unused-command-line-argument") -endif() - -# Find required dependencies -find_package(HDF5 REQUIRED COMPONENTS CXX) -find_package(Boost REQUIRED) +# Find aqnwb package. The aqnwb_DIR must be set on the command line +# e.g. -Daqnwb_DIR=/path/to/aqnwb/install/lib/cmake/aqnwb +find_package(aqnwb REQUIRED) # Generated source files set(GENERATED_SOURCES""" @@ -1055,29 +1038,13 @@ def generate_test_app_cmake(output_dir: Path, app_name: str, cpp_files: List[str # Include directories target_include_directories({app_name} PRIVATE - ${{AQNWB_SRC_DIR}} - ${{CMAKE_CURRENT_SOURCE_DIR}}/.. - ${{CMAKE_CURRENT_SOURCE_DIR}}/../spec - ${{HDF5_INCLUDE_DIRS}} - ${{Boost_INCLUDE_DIRS}} + "${{CMAKE_CURRENT_SOURCE_DIR}}/.." + "${{CMAKE_CURRENT_SOURCE_DIR}}/../spec" ) -# Find the aqnwb library -find_library(AQNWB_LIBRARY - NAMES aqnwb - HINTS "${{AQNWB_DIR}}" - PATH_SUFFIXES "lib" "bin" -) -if (NOT AQNWB_LIBRARY) - message(FATAL_ERROR "Could not find aqnwb library in ${{AQNWB_DIR}}. Please build the main project first or set AQNWB_DIR to the correct path.") -endif() - - # Link libraries target_link_libraries({app_name} - ${{AQNWB_LIBRARY}} - ${{HDF5_CXX_LIBRARIES}} - ${{Boost_LIBRARIES}} + aqnwb::aqnwb ) # If on Windows, link bcrypt From b1cd13ea699fc1760e9cf979ad8e67796b1c118a Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 30 Sep 2025 01:07:11 -0700 Subject: [PATCH 11/15] Path HDF5 and BOOST install paths --- .github/workflows/generate-classes.yml | 8 ++++++-- .gitignore | 4 +--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/generate-classes.yml b/.github/workflows/generate-classes.yml index 1e7d90a1e..31e437b6f 100644 --- a/.github/workflows/generate-classes.yml +++ b/.github/workflows/generate-classes.yml @@ -67,7 +67,9 @@ jobs: cd test_output/test_app mkdir build cd build - cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" ../ + HDF5_ROOT=$(brew --prefix hdf5) + BOOST_ROOT=$(brew --prefix boost) + cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" -DCMAKE_PREFIX_PATH="$HDF5_ROOT;$BOOST_ROOT" ../ make -j 2 - name: Run the test program for LabMetadataExample extension @@ -110,7 +112,9 @@ jobs: cd test_output_nwb_schema/test_app mkdir build cd build - cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" ../ + HDF5_ROOT=$(brew --prefix hdf5) + BOOST_ROOT=$(brew --prefix boost) + cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" -DCMAKE_PREFIX_PATH="$HDF5_ROOT;$BOOST_ROOT" ../ make -j 2 - name: Run the test program for the nwb_schema diff --git a/.gitignore b/.gitignore index 1bef69b97..77c13515c 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ __pycache__/ .Python env/ venv/ +.ipynb_checkpoints # Python packaging build/ @@ -46,9 +47,6 @@ dist/ *.egg-info/ wheels/ -# Jupyter Notebook -.ipynb_checkpoints - # Pytest .pytest_cache/ htmlcov/ From 9d0257865ff8fdf6a7128fa6506b01c93caf2769 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 30 Sep 2025 01:13:50 -0700 Subject: [PATCH 12/15] Add missing includes for HDF5 and BOOST --- .github/workflows/generate-classes.yml | 8 ++------ resources/utils/schematype_to_aqnwb.py | 9 +++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/generate-classes.yml b/.github/workflows/generate-classes.yml index 31e437b6f..1e7d90a1e 100644 --- a/.github/workflows/generate-classes.yml +++ b/.github/workflows/generate-classes.yml @@ -67,9 +67,7 @@ jobs: cd test_output/test_app mkdir build cd build - HDF5_ROOT=$(brew --prefix hdf5) - BOOST_ROOT=$(brew --prefix boost) - cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" -DCMAKE_PREFIX_PATH="$HDF5_ROOT;$BOOST_ROOT" ../ + cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" ../ make -j 2 - name: Run the test program for LabMetadataExample extension @@ -112,9 +110,7 @@ jobs: cd test_output_nwb_schema/test_app mkdir build cd build - HDF5_ROOT=$(brew --prefix hdf5) - BOOST_ROOT=$(brew --prefix boost) - cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" -DCMAKE_PREFIX_PATH="$HDF5_ROOT;$BOOST_ROOT" ../ + cmake -Daqnwb_DIR="${{ github.workspace }}/prefix/lib/cmake/aqnwb" ../ make -j 2 - name: Run the test program for the nwb_schema diff --git a/resources/utils/schematype_to_aqnwb.py b/resources/utils/schematype_to_aqnwb.py index 780052927..aa44662c7 100755 --- a/resources/utils/schematype_to_aqnwb.py +++ b/resources/utils/schematype_to_aqnwb.py @@ -1021,6 +1021,11 @@ def generate_test_app_cmake(output_dir: Path, app_name: str, cpp_files: List[str # e.g. -Daqnwb_DIR=/path/to/aqnwb/install/lib/cmake/aqnwb find_package(aqnwb REQUIRED) +# Find HDF5 +find_package(HDF5 REQUIRED COMPONENTS CXX) +# Find Boost +find_package(Boost REQUIRED) + # Generated source files set(GENERATED_SOURCES""" @@ -1040,11 +1045,15 @@ def generate_test_app_cmake(output_dir: Path, app_name: str, cpp_files: List[str target_include_directories({app_name} PRIVATE "${{CMAKE_CURRENT_SOURCE_DIR}}/.." "${{CMAKE_CURRENT_SOURCE_DIR}}/../spec" + ${{HDF5_INCLUDE_DIRS}} + ${{Boost_INCLUDE_DIRS}} ) # Link libraries target_link_libraries({app_name} aqnwb::aqnwb + ${{HDF5_CXX_LIBRARIES}} + ${{Boost_LIBRARIES}} ) # If on Windows, link bcrypt From 2e3847334368ba78dbf58ec58c593cf80c000aa2 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 1 Oct 2025 15:09:46 -0700 Subject: [PATCH 13/15] Uv install tools (#229) * Update to use uv for python tools * Update changelog --- .github/workflows/generate-classes.yml | 14 +++-- .github/workflows/python-utils.yml | 10 ++-- CHANGELOG.md | 9 ++-- docs/pages/devdocs/install.dox | 32 ++++++++++- resources/utils/README.md | 73 ++++++++++++++++++++++++++ resources/utils/aqnwb_utils.py | 6 +++ resources/utils/generate_spec_files.py | 6 +++ resources/utils/pyproject.toml | 25 +++++++++ resources/utils/schematype_to_aqnwb.py | 6 ++- resources/utils/setup.py | 17 ------ 10 files changed, 161 insertions(+), 37 deletions(-) create mode 100644 resources/utils/README.md create mode 100644 resources/utils/pyproject.toml delete mode 100644 resources/utils/setup.py diff --git a/.github/workflows/generate-classes.yml b/.github/workflows/generate-classes.yml index 1e7d90a1e..ec9de5c4d 100644 --- a/.github/workflows/generate-classes.yml +++ b/.github/workflows/generate-classes.yml @@ -40,10 +40,8 @@ jobs: with: python-version: '3.11' - - name: Install python script dependencies - run: | - python -m pip install --upgrade pip - pip install ./resources/utils + - name: Install uv + run: brew install uv # ----------------------------------------------------------------- # 2. LabMetadataExtension pipeline @@ -52,11 +50,11 @@ jobs: run: | mkdir test_output mkdir test_output/spec - aqnwb-utils generate-spec demo/labmetadata_extension_demo/spec test_output/spec + uv run resources/utils/aqnwb_utils.py generate-spec demo/labmetadata_extension_demo/spec test_output/spec - name: Run schematype_to_aqnwb.py to generate AqNWB classes for the LabMetadataExample extension run: | - aqnwb-utils generate-types --generate-test-app demo/labmetadata_extension_demo/spec/ndx-labmetadata-example.namespace.yaml test_output + uv run resources/utils/aqnwb_utils.py generate-types --generate-test-app demo/labmetadata_extension_demo/spec/ndx-labmetadata-example.namespace.yaml test_output - name: List generated files for the LabMetadataExample extension run: | @@ -95,11 +93,11 @@ jobs: - name: Run generate_nwb_schema_headers.sh to generated headers the nwb_schema run: | mkdir test_output_nwb_schema/spec - aqnwb-utils generate-spec test_output_nwb_schema/nwb-schema/core test_output_nwb_schema/spec + uv run resources/utils/aqnwb_utils.py generate-spec test_output_nwb_schema/nwb-schema/core test_output_nwb_schema/spec - name: Run schematype_to_aqnwb.py to generate AqNWB classes for the nwb_schema run: | - aqnwb-utils generate-types --generate-test-app test_output_nwb_schema/nwb-schema/core/nwb.namespace.yaml test_output_nwb_schema + uv run resources/utils/aqnwb_utils.py generate-types --generate-test-app test_output_nwb_schema/nwb-schema/core/nwb.namespace.yaml test_output_nwb_schema - name: List generated files for the nwb-schema run: | diff --git a/.github/workflows/python-utils.yml b/.github/workflows/python-utils.yml index 0adcec7dd..92d3016bd 100644 --- a/.github/workflows/python-utils.yml +++ b/.github/workflows/python-utils.yml @@ -19,10 +19,8 @@ jobs: with: python-version: '3.11' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install ./resources/utils + - name: Install uv + run: brew install uv - name: Clone latest NWB schema release run: | @@ -39,7 +37,7 @@ jobs: - name: Run generate_spec_files run: | - output=$(aqnwb-utils generate-spec test_output/nwb-schema/core test_output 2>&1) + output=$(uv run resources/utils/aqnwb_utils.py generate-spec test_output/nwb-schema/core test_output 2>&1) echo "$output" if echo "$output" | grep -q "ERROR"; then exit 1 @@ -47,7 +45,7 @@ jobs: - name: Run schematype_to_aqnwb run: | - output=$(aqnwb-utils generate-types test_output/nwb-schema/core/nwb.namespace.yaml test_output 2>&1) + output=$(uv run resources/utils/aqnwb_utils.py generate-types test_output/nwb-schema/core/nwb.namespace.yaml test_output 2>&1) echo "$output" if echo "$output" | grep -q "ERROR"; then exit 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 962d86d50..b7615a240 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,16 +8,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -* Added `aqnwb-utils` as a pip-installable command-line utility to provide a common interface for aqnwb command line tools, e.g., `schematype_to_aqnwb.py` and `generate_spec_files.py` (@oruebel, [#227](https://github.com/NeurodataWithoutBorders/aqnwb/pull/227)) - +* Python Utility enhancements: + * Added `aqnwb-utils` as a command-line utility to provide a common interface for aqnwb command line tools, e.g., `schematype_to_aqnwb.py` and `generate_spec_files.py`. (@oruebel, [#227](https://github.com/NeurodataWithoutBorders/aqnwb/pull/227)) + * Added inline script metadata (PEP 723) to Python utilities to enable direct execution with `uv run` without installation (@oruebel, [#229](https://github.com/NeurodataWithoutBorders/aqnwb/pull/229) + * Added `pyproject.toml` for modern Python packaging support (@oruebel, [#229](https://github.com/NeurodataWithoutBorders/aqnwb/pull/229) ### Changed +* Updated Python utilities to use `uv` instead of `pip` for dependency management and updated docs and github workflows to use uv (@oruebel, [#227](https://github.com/NeurodataWithoutBorders/aqnwb/pull/227) +* Updated documentation to refer to the new `aqnwb-utils` command-line utility (@oruebel, [#227](https://github.com/NeurodataWithoutBorders/aqnwb/pull/227)) * Enhanced the `schematype_to_aqnwb` utility script: * Generated source files are now placed into a folder hierarchy based on the name of the namespace and schemafile of the neurodata_type (@oruebel, [#224](https://github.com/NeurodataWithoutBorders/aqnwb/pull/224)) * Added functionality to optionally create a simple example app that instantiates all generated classes to help test that all generated classes can be compiled (@oruebel, [#225](https://github.com/NeurodataWithoutBorders/aqnwb/pull/225)) * Updated generation of header files to ensure proper compilation, e.g.: i) identify and include the headers of all neurodata_types that are being used, ii) fix formatting of comments to avoid nested multi-line comments, iii) fixed issues with incomplete typenames (@oruebel, [#225](https://github.com/NeurodataWithoutBorders/aqnwb/pull/225)) * Added GitHub action testing that all sources files generated by the `schematype_to_aqnwb` utility for the nwb-schema and LabMetaDataExtension example can be compiled (@oruebel, [#225](https://github.com/NeurodataWithoutBorders/aqnwb/pull/225)) -* Updated documentation to refer to the new `aqnwb-utils` command-line utility (@oruebel, [#226](https://github.com/NeurodataWithoutBorders/aqnwb/pull/226)) diff --git a/docs/pages/devdocs/install.dox b/docs/pages/devdocs/install.dox index 3118bad00..b5e771d50 100644 --- a/docs/pages/devdocs/install.dox +++ b/docs/pages/devdocs/install.dox @@ -27,13 +27,41 @@ * \section dev_utils_sec Python Utilities * * AqNWB provides a set of Python utilities for developers to help with generating C++ classes from NWB schema files. - * To install these utilities, run the following command from the root of the project: + * + * \subsection dev_utils_uv Using uv (Recommended) + * + * The utilities use inline script metadata (PEP 723) and can be run directly with `uv` without installation. + * First install `uv`: + * + * \code{.sh} + * brew install uv + * \endcode + * + * Then run the utilities directly: * * \code{.sh} + * uv run resources/utils/aqnwb_utils.py generate-spec + * uv run resources/utils/aqnwb_utils.py generate-types + * \endcode + * + * \subsection dev_utils_install Traditional Installation (Optional) + * + * Alternatively, you can install the utilities as a package: + * + * \code{.sh} + * # Using uv + * uv pip install ./resources/utils + * + * # Or using pip * pip install ./resources/utils * \endcode * - * This will install the `aqnwb-utils` command-line utility, which can be used to access the various Python utilities. + * After installation, the `aqnwb-utils` command will be available: + * + * \code{.sh} + * aqnwb-utils generate-spec + * aqnwb-utils generate-types + * \endcode * * \section devbuild_sec Developer Build * diff --git a/resources/utils/README.md b/resources/utils/README.md new file mode 100644 index 000000000..b831992d0 --- /dev/null +++ b/resources/utils/README.md @@ -0,0 +1,73 @@ +# AqNWB Utilities + +Command-line utilities for AqNWB development. + +## Installation + +### Using uv (Recommended) + +The utilities use inline script metadata (PEP 723) and can be run directly with `uv`: + +```bash +# Install uv if you haven't already +brew install uv + +# Run utilities directly without installation +uv run resources/utils/aqnwb_utils.py generate-spec +uv run resources/utils/aqnwb_utils.py generate-types +``` + +### Traditional Installation + +You can also install the package if you prefer: + +```bash +# Using uv +uv pip install ./resources/utils + +# Or using pip +pip install ./resources/utils + +# Then use the installed command +aqnwb-utils generate-spec +``` + +## Usage + +### Direct execution with uv (no installation required) + +```bash +# Generate spec files +uv run resources/utils/aqnwb_utils.py generate-spec + +# Generate neurodata types +uv run resources/utils/aqnwb_utils.py generate-types + +# Generate types with test app +uv run resources/utils/aqnwb_utils.py generate-types --generate-test-app +``` + +### After installation + +```bash +# Generate spec files +aqnwb-utils generate-spec + +# Generate neurodata types +aqnwb-utils generate-types + +# Generate types with test app +aqnwb-utils generate-types --generate-test-app +``` + +## Available Commands + +- `generate-spec`: Generate C++ header files from NWB schema files +- `generate-types`: Generate C++ classes from neurodata types defined in schema files + +## Benefits of using uv + +- **No installation required**: Run scripts directly with their dependencies automatically managed +- **Isolated environments**: Each script run uses its own isolated environment +- **Fast**: uv is significantly faster than pip for dependency resolution and installation +- **Reproducible**: Dependencies are specified in the script itself using PEP 723 metadata diff --git a/resources/utils/aqnwb_utils.py b/resources/utils/aqnwb_utils.py index 39d1328a1..e1f026c3d 100644 --- a/resources/utils/aqnwb_utils.py +++ b/resources/utils/aqnwb_utils.py @@ -1,3 +1,9 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.8" +# dependencies = ["hdmf", "pynwb", "ruamel.yaml"] +# /// + import argparse import generate_spec_files import schematype_to_aqnwb diff --git a/resources/utils/generate_spec_files.py b/resources/utils/generate_spec_files.py index 527baae21..9f08aa1aa 100644 --- a/resources/utils/generate_spec_files.py +++ b/resources/utils/generate_spec_files.py @@ -1,3 +1,9 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.8" +# dependencies = ["ruamel.yaml"] +# /// + import json import argparse from pathlib import Path diff --git a/resources/utils/pyproject.toml b/resources/utils/pyproject.toml new file mode 100644 index 000000000..fb1db67f1 --- /dev/null +++ b/resources/utils/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "aqnwb-utils" +version = "0.1.0" +description = "Command-line utilities for AqNWB development" +readme = "README.md" +requires-python = ">=3.8" +license = {text = "BSD-3-Clause"} +authors = [ + {name = "AqNWB Development Team"} +] +dependencies = [ + "hdmf", + "pynwb", + "ruamel.yaml", +] + +[project.scripts] +aqnwb-utils = "aqnwb_utils:main" + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/resources/utils/schematype_to_aqnwb.py b/resources/utils/schematype_to_aqnwb.py index aa44662c7..b78c1b95f 100755 --- a/resources/utils/schematype_to_aqnwb.py +++ b/resources/utils/schematype_to_aqnwb.py @@ -1,4 +1,8 @@ -#!/usr/bin/env python3 +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.8" +# dependencies = ["hdmf", "pynwb", "ruamel.yaml"] +# /// """ Script to generate C++ code from NWB schema files. diff --git a/resources/utils/setup.py b/resources/utils/setup.py deleted file mode 100644 index 17e87e5f8..000000000 --- a/resources/utils/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import setup, find_packages - -with open("requirements.txt") as f: - install_requires = f.read().strip().split("\n") - -setup( - name="aqnwb-utils", - version="0.1.0", - packages=find_packages(), - py_modules=["aqnwb_utils", "generate_spec_files", "schematype_to_aqnwb"], - install_requires=install_requires, - entry_points={ - "console_scripts": [ - "aqnwb-utils=aqnwb_utils:main", - ] - }, -) From 1e646aa9662079d9c3b6e5d7b8e43a8428f8cd96 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 1 Oct 2025 16:06:58 -0700 Subject: [PATCH 14/15] Update docs to use uv for the python utilities throughout --- docs/pages/devdocs/integrating_extensions.dox | 14 +++--- docs/pages/devdocs/nwb_schema.dox | 9 ++-- docs/pages/devdocs/python_utils.dox | 46 ++++++++++++++++--- docs/pages/devdocs/registered_types.dox | 16 +++---- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/docs/pages/devdocs/integrating_extensions.dox b/docs/pages/devdocs/integrating_extensions.dox index 4a84639b4..2999e10e3 100644 --- a/docs/pages/devdocs/integrating_extensions.dox +++ b/docs/pages/devdocs/integrating_extensions.dox @@ -9,10 +9,10 @@ * Integrating a new schema namespace with AqNWB (e.g., to support an extension to NWB) involves generating * the necessary specification files and ensuring that the namespace is registered with the * \ref AQNWB::SPEC::NamespaceRegistry "NamespaceRegistry". This process is simplified through - * the use of the `aqnwb-utils` command-line utility. + * the use of Python utilities that can be run with `uv`. * * @note - * See the \ref dev_utils_sec "Python Utilities" section in the developer installation guide for instructions on how to install the utilities. + * See the \ref dev_utils_sec "Python Utilities" section in the developer installation guide for instructions on how to use the utilities with `uv`. * * 1. **Get the schema files**: Download or create the schema for the namespace in YAML format. * @note @@ -20,13 +20,13 @@ * [NWB Extension Tutorial](https://nwb-overview.readthedocs.io/en/latest/extensions_tutorial/extensions_tutorial_home.html) * for more information on how to create data schema for NWB. * - * 2. **Convert the schema files to C++**: Run the `aqnwb-utils generate-spec` command on your + * 2. **Convert the schema files to C++**: Run the `generate-spec` command on your * schema files to generate the necessary C++ header files. This script processes the schema and creates * the appropriate C++ header files that include the namespace definitions and registration. * @note * To learn more about how to use the script and its parameters, you can view the help doc by running: * @code - * aqnwb-utils generate-spec --help + * uv run resources/utils/aqnwb_utils.py generate-spec --help * @endcode * * 3. **Include the Generated Header Files**: In your C++ code that uses AqNWB, include the generated header files. The @@ -37,7 +37,7 @@ * * 4. **Implement appropriate RegisteredType classes**: Follow the tutorial on * \ref registered_type_page to define appropriate interfaces for the `neurodata_type`s defined in your new namespace. - * The `aqnwb-utils generate-types` command can also provide additional help by providing a simple + * The `generate-types` command can also provide additional help by providing a simple * utility that can automatically generate skeleton AqNWB C++ classes for neurodata_types * directly from JSON/YAML schema files. * @@ -93,11 +93,11 @@ * * @subsection labmetadata_extension_cpp_generation Step 2: Convert the Schema to C++ * - * The schema files are converted to C++ using the `aqnwb-utils generate-spec` command: + * The schema files are converted to C++ using the `generate-spec` command: * * @code * mkdir demo/labmetadata_extension_demo/src - * aqnwb-utils generate-spec demo/labmetadata_extension_demo/spec demo/labmetadata_extension_demo/src + * uv run resources/utils/aqnwb_utils.py generate-spec demo/labmetadata_extension_demo/spec demo/labmetadata_extension_demo/src * @endcode * * This generates the following new header file in the `demo/labmetadata_extension_demo/src` folder: diff --git a/docs/pages/devdocs/nwb_schema.dox b/docs/pages/devdocs/nwb_schema.dox index 4e0220e30..77a306552 100644 --- a/docs/pages/devdocs/nwb_schema.dox +++ b/docs/pages/devdocs/nwb_schema.dox @@ -19,7 +19,7 @@ * * This script will: * - Clone the latest NWB and HDMF schema repositories into a temporary directory - * - Run `aqnwb-utils generate-spec` for both the NWB core and HDMF common schemas + * - Run `uv run resources/utils/aqnwb_utils.py generate-spec` for both the NWB core and HDMF common schemas * - Copy the generated C++ header files to `src/spec` * - Clean up all temporary files automatically * @@ -33,14 +33,17 @@ * PYTHON=python3 bash resources/utils/generate_nwb_schema_headers.sh * @endcode * - * For use with extensions and other advanced or custom use cases, developers may still run `aqnwb-utils generate-spec` directly. + * For use with extensions and other advanced or custom use cases, developers may still run the `generate-spec` command directly: + * @code{.sh} + * uv run resources/utils/aqnwb_utils.py generate-spec + * @endcode * * * \section dev_docs_updating_nwb_schema_section Updating the schema * * Currently, the version of the schema being used for development is fixed and stored in the `/resources/schema` folder. * Updating to a newer version of the schema requires: - * - Regeneration of the `spec` header files via `aqnwb-utils generate-spec` + * - Regeneration of the `spec` header files via the `generate-spec` command * - Update of existing `Container` classes and unit tests in AqNWB to match changes in the new schema compared to the previous schema * - Successful completion of all unit-test and round-trip testing with PyNWB and MatNWB * diff --git a/docs/pages/devdocs/python_utils.dox b/docs/pages/devdocs/python_utils.dox index 81bac8877..21d24a6da 100644 --- a/docs/pages/devdocs/python_utils.dox +++ b/docs/pages/devdocs/python_utils.dox @@ -4,21 +4,43 @@ * \tableofcontents * * AqNWB provides a set of Python utilities for developers to help with generating C++ classes from NWB schema files. - * These utilities are available as a command-line tool called `aqnwb-utils`. * - * \section install_utils_sec Installing the Utilities + * \section install_utils_sec Using the Utilities with uv (Recommended) * - * To install the Python utilities, run the following command from the root of the project: + * The utilities use inline script metadata (PEP 723) and can be run directly with `uv` without installation. + * First install `uv`: * * \code{.sh} + * brew install uv + * \endcode + * + * Then run the utilities directly: + * + * \code{.sh} + * uv run resources/utils/aqnwb_utils.py [options] + * \endcode + * + * \subsection install_utils_traditional_sec Traditional Installation (Optional) + * + * Alternatively, you can install the utilities as a package: + * + * \code{.sh} + * # Using uv + * uv pip install ./resources/utils + * + * # Or using pip * pip install ./resources/utils * \endcode * - * This will install the `aqnwb-utils` command-line utility, which can be used to access the various Python utilities. + * After installation, the `aqnwb-utils` command will be available: * - * \section using_utils_sec Using the Utilities + * \code{.sh} + * aqnwb-utils [options] + * \endcode + * + * \section using_utils_sec Available Commands * - * The `aqnwb-utils` command-line utility provides two main commands: + * The utilities provide two main commands: * * - `generate-spec`: This command generates C++ header files from NWB schema files. * - `generate-types`: This command generates skeleton C++ source files for integrating new neurodata_types with AqNWB. @@ -29,6 +51,12 @@ * * The `generate-spec` command is used to generate C++ header files from NWB schema files. * + * Using `uv run` (recommended): + * \code{.sh} + * uv run resources/utils/aqnwb_utils.py generate-spec + * \endcode + * + * Or if installed: * \code{.sh} * aqnwb-utils generate-spec * \endcode @@ -40,6 +68,12 @@ * * The `generate-types` command is used to generate skeleton C++ source files for integrating new neurodata_types with AqNWB. * + * Using `uv run` (recommended): + * \code{.sh} + * uv run resources/utils/aqnwb_utils.py generate-types + * \endcode + * + * Or if installed: * \code{.sh} * aqnwb-utils generate-types * \endcode diff --git a/docs/pages/devdocs/registered_types.dox b/docs/pages/devdocs/registered_types.dox index ecd484303..7ec5af29f 100644 --- a/docs/pages/devdocs/registered_types.dox +++ b/docs/pages/devdocs/registered_types.dox @@ -415,9 +415,9 @@ * in the \ref AQNWB::NWB::ElectrodesTable "ElectrodesTable" to read the `group_name` column * as `VectorData` with the data type already specified as `std::string` at compile time. * - * \section using_schematype_to_aqnwb Using the aqnwb-utils generate-types command + * \section using_schematype_to_aqnwb Using the generate-types command * - * The `aqnwb-utils generate-types` command, included in the + * The `generate-types` command, included in the * [AqNWB source repository](https://github.com/NeurodataWithoutBorders/aqnwb), * is a simple utility designed to create skeleton C++ source files for integrating new * neurodata_types with AqNWB. While the generated source files are only an outline and are not guaranteed to compile, @@ -427,7 +427,7 @@ * for all neurodata_types in the NWB schema: * * @code - * aqnwb-utils generate-types nwb-schema/core/nwb.namespace.yaml test_output + * uv run resources/utils/aqnwb_utils.py generate-types nwb-schema/core/nwb.namespace.yaml test_output * @endcode * * The generated files will be placed in the folder hierarchy based on the name of the namespace and @@ -439,17 +439,17 @@ * `--generate-test-app` option: * * @code - * aqnwb-utils generate-types --generate-test-app nwb-schema/core/nwb.namespace.yaml test_output + * uv run resources/utils/aqnwb_utils.py generate-types --generate-test-app nwb-schema/core/nwb.namespace.yaml test_output * @endcode * * To learn more about how to use the script and its parameters, you can view the help doc by running: * * @code - * aqnwb-utils generate-types --help + * uv run resources/utils/aqnwb_utils.py generate-types --help * @endcode * * \note - * The `aqnwb-utils generate-types` command uses `PyNWB` for parsing schema. Currently the script does not unload + * The `generate-types` command uses `PyNWB` for parsing schema. Currently the script does not unload * namespaces loaded by default by `PyNWB`. I.e., if you see a warning of the form: * \code * UserWarning: Ignoring cached namespace 'core' version 2.7.0 because version 2.8.0 is already loaded. @@ -459,8 +459,8 @@ * NWB `core` and `hdmf-common` namespaces. * * \note - * When generating the test app via `aqnwb-utils generate-types --generate-test-app` we also - * need to generate the schema headers files and save them in the `/spec` folder. + * When generating the test app via the `--generate-test-app` option we also + * need to generate the schema headers files and save them in the `/spec` folder. * * \section implement_registered_type_unit_tests Testing RegisteredTypes * From e409432b327428126d6dc2e296e146659747d92d Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 1 Oct 2025 16:54:29 -0700 Subject: [PATCH 15/15] Simplify docs for the python utilities by making the resources/utils/README.md the source of truth to avoid duplication --- docs/Doxyfile.in | 4 +- docs/pages/2_devdocs.dox | 1 - docs/pages/devdocs/install.dox | 62 +++----- docs/pages/devdocs/integrating_extensions.dox | 2 +- docs/pages/devdocs/python_utils.dox | 132 ------------------ 5 files changed, 20 insertions(+), 181 deletions(-) delete mode 100644 docs/pages/devdocs/python_utils.dox diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index fb25c0494..7df3209d3 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -26,7 +26,7 @@ EXPAND_ONLY_PREDEF = YES # Add sources INPUT = "@PROJECT_SOURCE_DIR@/src" "@PROJECT_SOURCE_DIR@/docs/pages" RECURSIVE = YES -EXAMPLE_PATH = "@PROJECT_SOURCE_DIR@/tests" "@PROJECT_SOURCE_DIR@/.github/CODE_OF_CONDUCT.md" "@PROJECT_SOURCE_DIR@/Legal.txt" "@PROJECT_SOURCE_DIR@/LICENSE" "@PROJECT_SOURCE_DIR@/demo/labmetadata_extension_demo/src" "@PROJECT_SOURCE_DIR@/CHANGELOG.md" +EXAMPLE_PATH = "@PROJECT_SOURCE_DIR@/tests" "@PROJECT_SOURCE_DIR@/.github/CODE_OF_CONDUCT.md" "@PROJECT_SOURCE_DIR@/Legal.txt" "@PROJECT_SOURCE_DIR@/LICENSE" "@PROJECT_SOURCE_DIR@/demo/labmetadata_extension_demo/src" "@PROJECT_SOURCE_DIR@/CHANGELOG.md" "@PROJECT_SOURCE_DIR@/resources/utils" IMAGE_PATH = "@PROJECT_SOURCE_DIR@/resources/images" EXTRACT_ALL = YES RECURSIVE = YES @@ -39,7 +39,7 @@ EXTRACT_STATIC = YES # HIDE_UNDOC_MEMBERS = YES # Enable Markdown support -MARKDOWN_SUPPORT = YES +MARKDOWN_SUPPORT = YES # Enable the call and caller graphs (this increases built time but seems reasonable for AqNWB) CALL_GRAPH = YES diff --git a/docs/pages/2_devdocs.dox b/docs/pages/2_devdocs.dox index cd6d5fa2b..9d2344424 100644 --- a/docs/pages/2_devdocs.dox +++ b/docs/pages/2_devdocs.dox @@ -4,7 +4,6 @@ * This documentation is intended for developers of AqNWB. * * - \subpage dev_install_page - * - \subpage python_utils_page * - \subpage testing * - \subpage dev_docs_page * - \subpage nwb_schema_page diff --git a/docs/pages/devdocs/install.dox b/docs/pages/devdocs/install.dox index b5e771d50..e361706d6 100644 --- a/docs/pages/devdocs/install.dox +++ b/docs/pages/devdocs/install.dox @@ -3,7 +3,9 @@ * * \tableofcontents * - * \section dev_requirements_sec Requirements + * \section dev_install_aqnwb_sec Installing AqNWB + * + * \subsection dev_requirements_sec Requirements * * Please ensure that the required libraries described in the * \ref user_requirements_sec "User Requirements" section are installed and @@ -24,46 +26,7 @@ * - clang-format (optional, required for ``target=format-check``, ``target=format-fix``) * - codespell (optional, required for ``target=spell-check``, ``target=spell-fix``) * - * \section dev_utils_sec Python Utilities - * - * AqNWB provides a set of Python utilities for developers to help with generating C++ classes from NWB schema files. - * - * \subsection dev_utils_uv Using uv (Recommended) - * - * The utilities use inline script metadata (PEP 723) and can be run directly with `uv` without installation. - * First install `uv`: - * - * \code{.sh} - * brew install uv - * \endcode - * - * Then run the utilities directly: - * - * \code{.sh} - * uv run resources/utils/aqnwb_utils.py generate-spec - * uv run resources/utils/aqnwb_utils.py generate-types - * \endcode - * - * \subsection dev_utils_install Traditional Installation (Optional) - * - * Alternatively, you can install the utilities as a package: - * - * \code{.sh} - * # Using uv - * uv pip install ./resources/utils - * - * # Or using pip - * pip install ./resources/utils - * \endcode - * - * After installation, the `aqnwb-utils` command will be available: - * - * \code{.sh} - * aqnwb-utils generate-spec - * aqnwb-utils generate-types - * \endcode - * - * \section devbuild_sec Developer Build + * \subsection devbuild_sec Developer Build * * Build system targets that are only useful for developers of AqNWB are * hidden if the `aqnwb_DEVELOPER_MODE` option is disabled. Enabling this @@ -96,7 +59,7 @@ * The use of `HDF5_ROOT` and `BOOST_ROOT` environment variables is deprecated for modern CMake * and may not work reliably with recent CMake versions. * - * \section devbuild_presets_subsec Developer Presets + * \subsubsection devbuild_presets_subsec Developer Presets * * As a developer, you can create your own dev preset by making a `CMakeUserPresets.json` file at the root of * the project: @@ -141,7 +104,7 @@ * Replace `` in the `CMakeUserPresets.json` file with the name of * the operating system you have (`win64`, `linux` or `darwin`). * - * \subsection configure_build_test Configure, Build and Test + * \subsubsection configure_build_test Configure, Build and Test * * You can configure, build and test the project respectively with the following commands from the project root on * any operating system with any build system: @@ -152,7 +115,7 @@ * ctest --preset=dev * \endcode * - * \section devbuild_dev_mode_targets_subsec Developer Mode Targets + * \subsubsection devbuild_dev_mode_targets_subsec Developer Mode Targets * * Additional targets can be invoked when in development mode using the commands below * @@ -160,10 +123,19 @@ * cmake --build --preset=dev --target= * \endcode * - * \subsection devbuild_target_options_subsubsec Target options + * \paragraph devbuild_target_options_subsubsec Target options * - `format-check`: run the `clang-format` tool on the codebase to check for formatting errors * - `format-fix` : run the `clang-format` tool on the codebase with `FIX=YES` to both check and automatically fix for formatting errors * - `spell-check`: run the `codespell` tool on the codebase to check for common spelling errors * - `spell-fix` : run the `codespell` tool on the codebase with `FIX=YES` to both check and automatically fix common spelling errors * - `docs` : builds the documentation using Doxygen. (Note: run `cmake --preset=dev -DBUILD_DOCS=ON` before building to add docs target) + * + * \section dev_install_utils_sec Installing Python Utilities + * + * AqNWB provides a set of Python utilities for developers to help with generating C++ classes from NWB schema files. + * For details on how to use the utilities, see the [README.md](https://github.com/NeurodataWithoutBorders/aqnwb/blob/main/resources/utils/README.md) + * in the `resources/utils` directory. + * + * **README.md** + * \include resources/utils/README.md */ diff --git a/docs/pages/devdocs/integrating_extensions.dox b/docs/pages/devdocs/integrating_extensions.dox index 2999e10e3..497870278 100644 --- a/docs/pages/devdocs/integrating_extensions.dox +++ b/docs/pages/devdocs/integrating_extensions.dox @@ -12,7 +12,7 @@ * the use of Python utilities that can be run with `uv`. * * @note - * See the \ref dev_utils_sec "Python Utilities" section in the developer installation guide for instructions on how to use the utilities with `uv`. + * See the \ref dev_install_utils_sec "Python Utilities" section in the developer installation guide for instructions on how to use the utilities with `uv`. * * 1. **Get the schema files**: Download or create the schema for the namespace in YAML format. * @note diff --git a/docs/pages/devdocs/python_utils.dox b/docs/pages/devdocs/python_utils.dox deleted file mode 100644 index 21d24a6da..000000000 --- a/docs/pages/devdocs/python_utils.dox +++ /dev/null @@ -1,132 +0,0 @@ -/** - * \page python_utils_page Python Utilities 🐍 - * - * \tableofcontents - * - * AqNWB provides a set of Python utilities for developers to help with generating C++ classes from NWB schema files. - * - * \section install_utils_sec Using the Utilities with uv (Recommended) - * - * The utilities use inline script metadata (PEP 723) and can be run directly with `uv` without installation. - * First install `uv`: - * - * \code{.sh} - * brew install uv - * \endcode - * - * Then run the utilities directly: - * - * \code{.sh} - * uv run resources/utils/aqnwb_utils.py [options] - * \endcode - * - * \subsection install_utils_traditional_sec Traditional Installation (Optional) - * - * Alternatively, you can install the utilities as a package: - * - * \code{.sh} - * # Using uv - * uv pip install ./resources/utils - * - * # Or using pip - * pip install ./resources/utils - * \endcode - * - * After installation, the `aqnwb-utils` command will be available: - * - * \code{.sh} - * aqnwb-utils [options] - * \endcode - * - * \section using_utils_sec Available Commands - * - * The utilities provide two main commands: - * - * - `generate-spec`: This command generates C++ header files from NWB schema files. - * - `generate-types`: This command generates skeleton C++ source files for integrating new neurodata_types with AqNWB. - * - * You can get more information about each command by running it with the `--help` flag. - * - * \subsection generate_spec_sec generate-spec - * - * The `generate-spec` command is used to generate C++ header files from NWB schema files. - * - * Using `uv run` (recommended): - * \code{.sh} - * uv run resources/utils/aqnwb_utils.py generate-spec - * \endcode - * - * Or if installed: - * \code{.sh} - * aqnwb-utils generate-spec - * \endcode - * - * - ``: The directory containing the NWB schema files. - * - ``: The directory where the generated C++ header files will be saved. - * - * \subsection generate_types_sec generate-types - * - * The `generate-types` command is used to generate skeleton C++ source files for integrating new neurodata_types with AqNWB. - * - * Using `uv run` (recommended): - * \code{.sh} - * uv run resources/utils/aqnwb_utils.py generate-types - * \endcode - * - * Or if installed: - * \code{.sh} - * aqnwb-utils generate-types - * \endcode - * - * - ``: The path to the namespace file (e.g., `nwb.namespace.yaml`). - * - ``: The directory where the generated C++ source files will be saved. - * - * \section create_utils_sec Creating a New Utility - * - * To create a new utility, you need to: - * - * 1. Create a new Python file in the `resources/utils` directory. - * 2. In the new file, define a function that takes `argparse.ArgumentParser` as input and adds the necessary arguments for your utility. - * 3. In the same file, define a function that takes the parsed arguments as input and implements the logic for your utility. - * 4. In `resources/utils/aqnwb_utils.py`, import your new functions and add a new subcommand to the `subparsers` object. - * - * For example, to add a new command called `my-command`, you would: - * - * 1. Create a new file `resources/utils/my_command.py` with the following content: - * - * \code{.py} - * def setup_parser(parser): - * parser.add_argument("my_arg", help="My argument") - * - * def main(args): - * print(f"My argument is {args.my_arg}") - * - * if __name__ == "__main__": - * import argparse - * parser = argparse.ArgumentParser() - * setup_parser(parser) - * args = parser.parse_args() - * main(args) - * \endcode - * - * 2. In `resources/utils/aqnwb_utils.py`, add the following: - * - * \code{.py} - * from . import my_command - * - * # ... - * - * def main(): - * # ... - * subparsers = parser.add_subparsers(dest="command") - * # ... - * my_command_parser = subparsers.add_parser("my-command", help="My command") - * my_command.setup_parser(my_command_parser) - * - * # ... - * - * if args.command == "my-command": - * my_command.main(args) - * \endcode - * - */