Adds Python bindings to Adiak

.github/workflows/cmake.yml

@@ -25,24 +25,59 @@ jobs:
           - {
               name: "Ubuntu Latest GCC",
               os: ubuntu-latest,
-              build_type: "Release", cc: "gcc", cxx: "g++"
+              build_type: "Release", cc: "gcc", cxx: "g++",
+              mpi_type: "none"
             }
           - {
               name: "Ubuntu Latest Clang",
               os: ubuntu-latest,
-              build_type: "Release", cc: "clang", cxx: "clang++"
+              build_type: "Release", cc: "clang", cxx: "clang++",
+              mpi_type: "none"
+            }
+          - {
+              name: "Ubuntu Latest GCC (MPI)",
+              os: ubuntu-latest,
+              build_type: "Release", cc: "gcc", cxx: "g++",
+              mpi_type: "mpich"
+            }
+          - {
+              name: "Ubuntu Latest Clang (MPI)",
+              os: ubuntu-latest,
+              build_type: "Release", cc: "clang", cxx: "clang++",
+              mpi_type: "mpich"
             }
           - {
               name: "macOS Latest Clang",
               os: macos-latest,
-              build_type: "Release", cc: "clang", cxx: "clang++"
+              build_type: "Release", cc: "clang", cxx: "clang++",
+              mpi_type: "none"
             }
     steps:
     - name: Checkout repository and submodules
       uses: actions/checkout@v2
       with:
         submodules: recursive
 
+    - name: Install Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.13'
+
+    - name: Install MPI
+      if: ${{ matrix.config.mpi_type != 'none' }}
+      uses: mpi4py/[email protected]
+      with:
+        mpi: ${{matrix.config.mpi_type}}
+
+    - name: Install Python binding dependencies
+      run: |
+        python3 -m pip install pybind11
+
+    - name: Install mpi4py
+      if: ${{ matrix.config.mpi_type != 'none' }}
+      run: |
+        python -m pip install mpi4py
+
     - name: Create Build Environment
       # Some projects don't allow in-source building, so create a separate build directory
       # We'll use this as our working directory for all subsequent commands
@@ -56,7 +91,13 @@ jobs:
       # Note the current convention is to use the -S and -B options here to specify source 
       # and build directories, but this is only available with CMake 3.13 and higher.  
       # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
-      run: cmake $GITHUB_WORKSPACE -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_C_COMPILER=${{matrix.config.cc}} -DCMAKE_CXX_COMPILER=${{matrix.config.cxx}}
+      run: |
+        EXTRA_OPTS=""
+        if [ "${{ matrix.config.mpi_type }}" != "none" ]; then
+          EXTRA_OPTS="-DENABLE_MPI=ON"
+        fi
+        cmake $GITHUB_WORKSPACE -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_C_COMPILER=${{matrix.config.cc}} -DCMAKE_CXX_COMPILER=${{matrix.config.cxx}} -DENABLE_PYTHON_BINDINGS=ON -Dpybind11_DIR=$(pybind11-config --cmakedir) \
+          $EXTRA_OPTS
 
     - name: Build
       working-directory: ${{github.workspace}}/build

CMakeLists.txt

@@ -9,6 +9,11 @@ if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
   set(CMAKE_INSTALL_LIBDIR lib)
 endif()
 
+# Find Python by location
+if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.15)
+  cmake_policy(SET CMP0094 NEW)
+endif()
+
 set(BLT_EXPORT_THIRDPARTY ON CACHE BOOL "")
 
 if (DEFINED BLT_SOURCE_DIR)
@@ -31,6 +36,19 @@ endif()
 
 include(${BLT_SOURCE_DIR}/SetupBLT.cmake)
 
+option(ENABLE_PYTHON_BINDINGS "Build/Install Python bindings for Adiak" FALSE)
+
+if (ENABLE_PYTHON_BINDINGS)
+  find_package(Python COMPONENTS Interpreter Development REQUIRED)
+  find_package(pybind11 CONFIG REQUIRED)
+
+  if (pybind11_FOUND)
+    set(ADIAK_HAVE_PYBIND11 TRUE)
+  else()
+    message(WARNING "Python bindings requested, but pybind11 not found")
+  endif()
+endif()
+
 add_subdirectory(src)
 if (ENABLE_TESTS)
   add_subdirectory(tests)

cmake/get_python_install_paths.py

@@ -0,0 +1,14 @@
+import sys
+import sysconfig
+
+if len(sys.argv) != 3 or sys.argv[1] not in ("purelib", "platlib"):
+    raise RuntimeError(
+        "Usage: python get_python_install_paths.py <purelib | platlib> <sysconfig_scheme>"
+    )
+
+install_dir = sysconfig.get_path(sys.argv[1], sys.argv[2], {"userbase": "", "base": ""})
+
+if install_dir.startswith("/"):
+    install_dir = install_dir[1:]
+
+print(install_dir, end="")

docs/sphinx/PythonSupport.rst

@@ -0,0 +1,106 @@
+Python support
+==============
+
+Adiak provides Python bindings based on `pybind11 <https://pybind11.readthedocs.io/en/stable/>`_
+for the metadata/annotation APIs (``init``, ``value``, ``fini``) and a set of convenience
+collectors (e.g., ``collect_all()``, ``walltime()``).
+To build Adiak with Python support, enable
+the :code:`ENABLE_PYTHON_BINDINGS` option in the CMake configuration:
+
+.. code-block:: sh
+
+    $ cmake -DENABLE_PYTHON_BINDINGS=ON ..
+
+If you want to initialize Adiak with an MPI communicator from Python, also enable (or auto-detect)
+MPI at configure time (mpi4py is only needed at runtime if you actually pass a communicator):
+
+.. code-block:: sh
+
+    $ cmake -DENABLE_PYTHON_BINDINGS=ON -DENABLE_MPI=ON ..
+
+Using the Python module
+-----------------------
+
+The Python module requires pybind11 and an installation of Python that both supports
+pybind11 and provides development headers (e.g., :code:`Python.h`) and libraries
+(e.g., :code:`libpython3.8.so`).
+
+The Adiak Python module is installed in either :code:`lib/pythonX.Y/site-packages/` and/or
+:code:`lib64/pythonX.Y/site-packages` in the Adiak installation directory. In these paths,
+:code:`X.Y` corresponds to the major and minor version numbers of the Python installation used.
+Additionally, :code:`lib/` and :code:`lib64/` will be used in accordance with the configuration
+of the Python installed.
+
+To use the Adiak Python module, simply add the directories above to :code:`PYTHONPATH` or
+:code:`sys.path`. Note that the module will be automatically added to :code:`PYTHONPATH` when
+loading the Adiak package with Spack if the :code:`python` variant is enabled.
+The module can then be imported with :code:`import pyadiak`.
+
+Example: basic usage (serial)
+-----------------------------
+
+.. code-block:: python
+
+    from datetime import datetime
+    from pathlib import Path
+
+    from pyadiak.annotations import init, value, fini, collect_all, walltime, cputime, systime
+    from pyadiak.types import Version, Path as APath, CatStr, Category
+
+    def main():
+        init(None)  # serial mode
+
+        value("str", "s")
+        value("compiler", Version("[email protected]"))
+        value("mydouble", 3.14)
+        value("problemsize", 14000, category=Category.Tuning)
+        value("countdown", 9876543210)
+
+        grid = [4.5, 1.18, 0.24, 8.92]
+        value("gridvalues", grid)
+
+        names = {"bob", "jim", "greg"}
+        value("allnames", names)
+
+        # Flatten nested structures (or use JsonStr)
+        names_arr = ["first", "second", "third"]
+        xs = [1.0, 2.0, 3.0]
+        ys = [1.0, 4.0, 9.0]
+        value("points.names", names_arr)
+        value("points.x", xs)
+        value("points.y", ys)
+
+        # Time & paths are auto-wrapped; shown explicitly here for clarity
+        value("birthday", datetime.fromtimestamp(286551000))   # Timepoint
+        value("nullpath", APath(Path("/dev/null")))            # Path
+        value("githash", CatStr("a0c93767478f23602c2eb317f641b091c52cf374"))
+
+        collect_all()
+        walltime()
+        cputime()
+        systime()
+
+        fini()
+
+    if __name__ == "__main__":
+        main()
+
+
+Example: MPI usage (optional)
+-----------------------------
+
+If built with :code:`-DENABLE_MPI=ON` and :code:`mpi4py` is available:
+
+.. code-block:: python
+
+    from mpi4py import MPI
+    from pyadiak.annotations import init, value, fini
+
+    def main():
+        comm = MPI.COMM_WORLD
+        init(comm)           # initialize with communicator
+        value("rank", comm.Get_rank())
+        fini()
+
+    if __name__ == "__main__":
+        main()

docs/sphinx/index.rst

@@ -33,6 +33,7 @@ by printf-inspired strings.
    ToolsUsingAdiak
    ApplicationAPI
    ToolAPI
+   PythonSupport
 
 Contributing
 ==================

src/CMakeLists.txt

@@ -53,3 +53,10 @@ install(TARGETS
   ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
 
 install(EXPORT adiak-targets NAMESPACE adiak:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/adiak)
+
+if (ENABLE_PYTHON_BINDINGS)
+  add_subdirectory(interface/python)
+  if (MPI_FOUND)
+    find_package(MPI REQUIRED CXX)
+  endif()
+endif()

src/interface/python/CMakeLists.txt

@@ -0,0 +1,68 @@
+set(PYADIAK_BINDING_SOURCES
+    mod.cpp
+    pyadiak_tool.cpp
+    pyadiak.cpp
+    types.cpp
+)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
+
+set(PYADIAK_SYSCONFIG_SCHEME "posix_user" CACHE STRING "Scheme used for searching for pyadiak's install path. Valid options can be determined with 'sysconfig.get_scheme_names()'")
+
+execute_process(
+  COMMAND
+    ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/get_python_install_paths.py purelib ${PYADIAK_SYSCONFIG_SCHEME}
+  OUTPUT_VARIABLE
+    PYADIAK_SITELIB)
+execute_process(
+  COMMAND
+    ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/get_python_install_paths.py platlib ${PYADIAK_SYSCONFIG_SCHEME}
+  OUTPUT_VARIABLE
+    PYADIAK_SITEARCH)
+
+set(PYADIAK_SITELIB "${PYADIAK_SITELIB}/pyadiak")
+set(PYADIAK_SITEARCH "${PYADIAK_SITEARCH}/pyadiak")
+
+pybind11_add_module(__pyadiak_impl ${PYADIAK_BINDING_SOURCES})
+target_link_libraries(__pyadiak_impl PUBLIC adiak)
+target_compile_features(__pyadiak_impl PUBLIC cxx_std_11)
+target_include_directories(__pyadiak_impl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+execute_process(
+  COMMAND python3 -c "import mpi4py, os; print(os.path.join(os.path.dirname(mpi4py.__file__), 'include'))"
+  OUTPUT_VARIABLE MPI4PY_INCLUDE_DIR
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+include_directories(${MPI4PY_INCLUDE_DIR})
+
+add_custom_target(
+    pyadiak_py_source_copy ALL # Always build pycaliper_test
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/pyadiak
+    COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/pyadiak ${CMAKE_CURRENT_BINARY_DIR}/pyadiak
+    COMMENT "Copying pyadiak Python source to ${CMAKE_CURRENT_BINARY_DIR}/pyadiak"
+)
+add_dependencies(__pyadiak_impl pyadiak_py_source_copy)
+
+install(
+    DIRECTORY
+        pyadiak/
+    DESTINATION
+        ${PYADIAK_SITELIB}
+)
+
+install(
+    TARGETS
+        __pyadiak_impl
+    ARCHIVE DESTINATION
+        ${PYADIAK_SITEARCH}
+    LIBRARY DESTINATION
+        ${PYADIAK_SITEARCH}
+)
+
+# Put the compiled extension inside the pyadiak package in the build tree
+set(_py_pkg_dir "${CMAKE_BINARY_DIR}/src/interface/python/pyadiak")
+
+set_target_properties(__pyadiak_impl PROPERTIES
+  LIBRARY_OUTPUT_DIRECTORY  "${_py_pkg_dir}"   # where the .so (MODULE/LIBRARY) goes
+  RUNTIME_OUTPUT_DIRECTORY  "${_py_pkg_dir}"
+  ARCHIVE_OUTPUT_DIRECTORY  "${_py_pkg_dir}"
+)

src/interface/python/common.hpp

@@ -0,0 +1,12 @@
+#ifndef ADIAK_INTERFACE_PYTHON_COMMON_HPP_
+#define ADIAK_INTERFACE_PYTHON_COMMON_HPP_
+
+#include <adiak.hpp>
+#include <adiak_internal.hpp>
+#include <adiak_tool.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+namespace py = pybind11;
+
+#endif /* ADIAK_INTERFACE_PYTHON_COMMON_HPP_ */

src/interface/python/mod.cpp

@@ -0,0 +1,17 @@
+#include "pyadiak.hpp"
+#include "pyadiak_tool.hpp"
+
+PYBIND11_MODULE(__pyadiak_impl, m) {
+  // TODO add version
+
+  auto types_module =
+      m.def_submodule("types", "Submodule for type wrappers in Adiak");
+  adiak::python::create_adiak_types_mod(types_module);
+
+  auto annotation_module =
+      m.def_submodule("annotations", "Submodule for annotation APIs");
+  adiak::python::create_adiak_annotation_mod(annotation_module);
+
+  auto tool_module = m.def_submodule("tools", "Submodule for tool APIs");
+  adiak::python::create_adiak_tool_mod(tool_module);
+}