From d3ec350bc4daaf32ebad7f6349b7ba8363d34fe1 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Thu, 11 Jul 2024 23:31:45 -0700 Subject: [PATCH 01/12] Implements Python bindings for Caliper --- CMakeLists.txt | 16 ++- bindings/python/CMakeLists.txt | 44 +++++++ bindings/python/annotation.cpp | 117 +++++++++++++++++ bindings/python/annotation.h | 40 ++++++ bindings/python/common.h | 12 ++ bindings/python/config_manager.cpp | 90 +++++++++++++ bindings/python/config_manager.h | 56 ++++++++ bindings/python/instrumentation.cpp | 119 +++++++++++++++++ bindings/python/instrumentation.h | 50 +++++++ bindings/python/loop.cpp | 27 ++++ bindings/python/loop.h | 26 ++++ bindings/python/mod.cpp | 69 ++++++++++ bindings/python/pycaliper/__init__.py | 15 +++ bindings/python/pycaliper/annotation.py | 1 + bindings/python/pycaliper/config_manager.py | 1 + bindings/python/pycaliper/high_level.py | 44 +++++++ bindings/python/pycaliper/instrumentation.py | 1 + bindings/python/pycaliper/loop.py | 1 + bindings/python/pycaliper/types.py | 1 + bindings/python/pycaliper/variant.py | 1 + bindings/python/variant.cpp | 130 +++++++++++++++++++ bindings/python/variant.h | 42 ++++++ cmake/get_python_install_paths.py | 14 ++ 23 files changed, 914 insertions(+), 3 deletions(-) create mode 100644 bindings/python/CMakeLists.txt create mode 100644 bindings/python/annotation.cpp create mode 100644 bindings/python/annotation.h create mode 100644 bindings/python/common.h create mode 100644 bindings/python/config_manager.cpp create mode 100644 bindings/python/config_manager.h create mode 100644 bindings/python/instrumentation.cpp create mode 100644 bindings/python/instrumentation.h create mode 100644 bindings/python/loop.cpp create mode 100644 bindings/python/loop.h create mode 100644 bindings/python/mod.cpp create mode 100644 bindings/python/pycaliper/__init__.py create mode 100644 bindings/python/pycaliper/annotation.py create mode 100644 bindings/python/pycaliper/config_manager.py create mode 100644 bindings/python/pycaliper/high_level.py create mode 100644 bindings/python/pycaliper/instrumentation.py create mode 100644 bindings/python/pycaliper/loop.py create mode 100644 bindings/python/pycaliper/types.py create mode 100644 bindings/python/pycaliper/variant.py create mode 100644 bindings/python/variant.cpp create mode 100644 bindings/python/variant.h create mode 100644 cmake/get_python_install_paths.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 7806d93d1..f7779eb93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,8 +50,9 @@ option(BUILD_SHARED_LIBS "Build shared libraries" TRUE) option(CMAKE_INSTALL_RPATH_USE_LINK_PATH "Add rpath for all dependencies" TRUE) # Optional Fortran -add_caliper_option(WITH_FORTRAN "Install Fortran interface" FALSE) -add_caliper_option(WITH_TOOLS "Build Caliper tools" TRUE) +add_caliper_option(WITH_FORTRAN "Install Fortran interface" FALSE) +add_caliper_option(WITH_PYTHON_BINDINGS "Install Python bindings" FALSE) +add_caliper_option(WITH_TOOLS "Build Caliper tools" TRUE) add_caliper_option(WITH_NVTX "Enable NVidia nvtx bindings for NVprof and NSight (requires CUDA)" FALSE) add_caliper_option(WITH_CUPTI "Enable CUPTI service (CUDA performance analysis)" FALSE) @@ -421,7 +422,12 @@ if(WITH_TAU) endif() # Find Python -find_package(Python COMPONENTS Interpreter REQUIRED) +set(FIND_PYTHON_COMPONENTS "Interpreter") +if (WITH_PYTHON_BINDINGS) + set(FIND_PYTHON_COMPONENTS "Development" ${FIND_PYTHON_COMPONENTS}) +endif () + +find_package(Python COMPONENTS ${FIND_PYTHON_COMPONENTS} REQUIRED) set(CALI_PYTHON_EXECUTABLE Python::Interpreter) if (WITH_SAMPLER) @@ -500,6 +506,10 @@ add_subdirectory(src) add_subdirectory(examples/apps EXCLUDE_FROM_ALL) +if (WITH_PYTHON_BINDINGS) + add_subdirectory(bindings/python) +endif () + if (BUILD_TESTING) add_subdirectory(test) endif() diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt new file mode 100644 index 000000000..efccb982f --- /dev/null +++ b/bindings/python/CMakeLists.txt @@ -0,0 +1,44 @@ +set(PYCALIPER_BINDING_SOURCES + annotation.cpp + config_manager.cpp + instrumentation.cpp + loop.cpp + variant.cpp + mod.cpp +) + +set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) + +find_package(pybind11 CONFIG REQUIRED) + +set(PYCALIPER_SYSCONFIG_SCHEME "posix_user" CACHE STRING "Scheme used for searching for pycaliper's install path. Valid options can be determined with 'sysconfig.get_scheme_names()'") + +execute_process(COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/get_python_install_paths.py purelib ${PYCALIPER_SYSCONFIG_SCHEME} OUTPUT_VARIABLE PYCALIPER_SITELIB) +execute_process(COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/get_python_install_paths.py platlib ${PYCALIPER_SYSCONFIG_SCHEME} OUTPUT_VARIABLE PYCALIPER_SITEARCH) + +message(STATUS "Pycaliper sitelib: ${PYCALIPER_SITELIB}") +message(STATUS "Pycaliper sitearch: ${PYCALIPER_SITEARCH}") + +set(PYCALIPER_SITELIB "${PYCALIPER_SITELIB}/pycaliper") +set(PYCALIPER_SITEARCH "${PYCALIPER_SITEARCH}/pycaliper") + +pybind11_add_module(__pycaliper_impl ${PYCALIPER_BINDING_SOURCES}) +target_link_libraries(__pycaliper_impl PUBLIC caliper) +target_compile_features(__pycaliper_impl PUBLIC cxx_std_11) +target_include_directories(__pycaliper_impl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + +install( + DIRECTORY + pycaliper/ + DESTINATION + ${PYCALIPER_SITELIB} +) + +install( + TARGETS + __pycaliper_impl + ARCHIVE DESTINATION + ${PYCALIPER_SITEARCH} + LIBRARY DESTINATION + ${PYCALIPER_SITEARCH} +) \ No newline at end of file diff --git a/bindings/python/annotation.cpp b/bindings/python/annotation.cpp new file mode 100644 index 000000000..8af1d7d01 --- /dev/null +++ b/bindings/python/annotation.cpp @@ -0,0 +1,117 @@ +#include "annotation.h" + +namespace cali { + +PythonAnnotation::PythonAnnotation(const char *name, cali_attr_properties opt) + : cali::Annotation(name, opt) {} + +PythonAnnotation &PythonAnnotation::begin() { + cali::Annotation::begin(); + return *this; +} + +PythonAnnotation &PythonAnnotation::begin(int data) { + cali::Annotation::begin(data); + return *this; +} + +PythonAnnotation &PythonAnnotation::begin(double data) { + cali::Annotation::begin(data); + return *this; +} + +PythonAnnotation &PythonAnnotation::begin(const char *data) { + cali::Annotation::begin(data); + return *this; +} + +PythonAnnotation &PythonAnnotation::begin(cali_attr_type type, + const std::string &data) { + cali::Annotation::begin(type, data.data(), data.size()); + return *this; +} + +PythonAnnotation &PythonAnnotation::begin(PythonVariant &data) { + cali::Annotation::begin(data); + return *this; +} + +PythonAnnotation &PythonAnnotation::set(int data) { + cali::Annotation::set(data); + return *this; +} + +PythonAnnotation &PythonAnnotation::set(double data) { + cali::Annotation::set(data); + return *this; +} + +PythonAnnotation &PythonAnnotation::set(const char *data) { + cali::Annotation::set(data); + return *this; +} + +PythonAnnotation &PythonAnnotation::set(cali_attr_type type, + const std::string &data) { + cali::Annotation::set(type, data.data(), data.size()); + return *this; +} + +PythonAnnotation &PythonAnnotation::set(PythonVariant &data) { + cali::Annotation::set(data); + return *this; +} + +void create_caliper_annotation_mod(py::module_ &caliper_annotation_mod) { + py::class_ annotation_type(caliper_annotation_mod, + "Annotation"); + annotation_type.def(py::init(), "", + py::arg(), py::arg("opt") = 0); + annotation_type.def("end", &PythonAnnotation::end, ""); + annotation_type.def("begin", + static_cast( + &PythonAnnotation::begin), + ""); + annotation_type.def("begin", + static_cast( + &PythonAnnotation::begin), + ""); + annotation_type.def( + "begin", + static_cast( + &PythonAnnotation::begin), + ""); + annotation_type.def( + "begin", + static_cast( + &PythonAnnotation::begin), + ""); + annotation_type.def( + "begin", + static_cast( + &PythonAnnotation::begin), + ""); + annotation_type.def("set", + static_cast( + &PythonAnnotation::set), + ""); + annotation_type.def( + "set", + static_cast( + &PythonAnnotation::set), + ""); + annotation_type.def( + "set", + static_cast( + &PythonAnnotation::set), + ""); + annotation_type.def( + "set", + static_cast( + &PythonAnnotation::set), + ""); +} + +} // namespace cali \ No newline at end of file diff --git a/bindings/python/annotation.h b/bindings/python/annotation.h new file mode 100644 index 000000000..0c15a523f --- /dev/null +++ b/bindings/python/annotation.h @@ -0,0 +1,40 @@ +#ifndef CALI_INTERFACE_PYTHON_ANNOTATION_H +#define CALI_INTERFACE_PYTHON_ANNOTATION_H + +#include "variant.h" + +namespace cali { + +class PythonAnnotation : public cali::Annotation { +public: + PythonAnnotation(const char *name, + cali_attr_properties opt = CALI_ATTR_DEFAULT); + + PythonAnnotation &begin(); + + PythonAnnotation &begin(int data); + + PythonAnnotation &begin(double data); + + PythonAnnotation &begin(const char *data); + + PythonAnnotation &begin(cali_attr_type type, const std::string &data); + + PythonAnnotation &begin(PythonVariant &data); + + PythonAnnotation &set(int data); + + PythonAnnotation &set(double data); + + PythonAnnotation &set(const char *data); + + PythonAnnotation &set(cali_attr_type type, const std::string &data); + + PythonAnnotation &set(PythonVariant &data); +}; + +void create_caliper_annotation_mod(py::module_ &caliper_annotation_mod); + +} // namespace cali + +#endif /* CALI_INTERFACE_PYTHON_ANNOTATION_H */ \ No newline at end of file diff --git a/bindings/python/common.h b/bindings/python/common.h new file mode 100644 index 000000000..dfcebd539 --- /dev/null +++ b/bindings/python/common.h @@ -0,0 +1,12 @@ +#ifndef CALI_INTERFACE_PYTHON_COMMON_HPP +#define CALI_INTERFACE_PYTHON_COMMON_HPP + +#include + +#include + +#include + +namespace py = pybind11; + +#endif /* CALI_INTERFACE_PYTHON_COMMON_HPP */ \ No newline at end of file diff --git a/bindings/python/config_manager.cpp b/bindings/python/config_manager.cpp new file mode 100644 index 000000000..633c57063 --- /dev/null +++ b/bindings/python/config_manager.cpp @@ -0,0 +1,90 @@ +#include "config_manager.h" +#include + +namespace cali { + +PythonConfigManager::PythonConfigManager() : cali::ConfigManager() {} + +PythonConfigManager::PythonConfigManager(const char *config_str) + : cali::ConfigManager(config_str) {} + +void PythonConfigManager::add_config_spec(py::dict json) { + add_config_spec(py::str(json)); +} + +void PythonConfigManager::add_option_spec(py::dict json) { + add_option_spec(py::str(json)); +} + +void PythonConfigManager::py_add(const char *config_string) { + cali::ConfigManager::add(config_string); +} + +void PythonConfigManager::check(const char *config_str) { + std::string err_msg = cali::ConfigManager::check(config_str); + if (err_msg.size() != 0) { + throw std::runtime_error(err_msg); + } +} + +void create_caliper_config_manager_mod( + py::module_ &caliper_config_manager_mod) { + py::class_ config_mgr_type(caliper_config_manager_mod, + "ConfigManager"); + config_mgr_type.def(py::init<>()); + config_mgr_type.def(py::init()); + config_mgr_type.def("add_config_spec", + static_cast( + &cali::ConfigManager::add_config_spec)); + config_mgr_type.def("add_config_spec", + static_cast( + &PythonConfigManager::add_config_spec)); + config_mgr_type.def("add_option_spec", + static_cast( + &cali::ConfigManager::add_option_spec)); + config_mgr_type.def("add_option_spec", + static_cast( + &PythonConfigManager::add_option_spec)); + config_mgr_type.def("add", &PythonConfigManager::py_add); + config_mgr_type.def("load", + static_cast( + &cali::ConfigManager::load)); + config_mgr_type.def( + "set_default_parameter", + static_cast( + &cali::ConfigManager::set_default_parameter)); + config_mgr_type.def( + "set_default_parameter_for_config", + static_cast( + &cali::ConfigManager::set_default_parameter_for_config)); + config_mgr_type.def("error", + static_cast( + &cali::ConfigManager::error)); + config_mgr_type.def("error_msg", + static_cast( + &cali::ConfigManager::error_msg)); + config_mgr_type.def("__repr__", + static_cast( + &cali::ConfigManager::error_msg)); + config_mgr_type.def("start", static_cast( + &cali::ConfigManager::start)); + config_mgr_type.def("stop", static_cast( + &cali::ConfigManager::stop)); + config_mgr_type.def("flush", static_cast( + &cali::ConfigManager::flush)); + config_mgr_type.def("check", &PythonConfigManager::check); + config_mgr_type.def( + "available_config_specs", + static_cast (PythonConfigManager::*)() const>( + &cali::ConfigManager::available_config_specs)); + config_mgr_type.def( + "get_documentation_for_spec", + static_cast( + &cali::ConfigManager::get_documentation_for_spec)); + config_mgr_type.def_static("get_config_docstrings", + static_cast (*)()>( + &cali::ConfigManager::get_config_docstrings)); +} + +} // namespace cali \ No newline at end of file diff --git a/bindings/python/config_manager.h b/bindings/python/config_manager.h new file mode 100644 index 000000000..9f18ee7da --- /dev/null +++ b/bindings/python/config_manager.h @@ -0,0 +1,56 @@ +#ifndef CALI_INTERFACE_PYTHON_CONFIG_MANAGER_H +#define CALI_INTERFACE_PYTHON_CONFIG_MANAGER_H + +#include "common.h" + +#include + +namespace cali { +class PythonConfigManager : public cali::ConfigManager { +public: + PythonConfigManager(); + + PythonConfigManager(const char *config_str); + + // void add_config_spec(const char *json); + + void add_config_spec(py::dict json); + + // void add_option_spec(const char *json); + + void add_option_spec(py::dict json); + + void py_add(const char *config_string); + + // void load(const char *filename); + + // void set_default_parameter(const char *key, const char *value); + + // void set_default_parameter_for_config(const char *config, const char *key, + // const char *value); + + // bool error() const; + + // Set to __repr__ + // bool error_msg() const; + + // void start(); + + // void stop(); + + // void flush(); + + void check(const char *config_str); + + // std::vector available_config_specs() const; + + // std::string get_documentation_for_spec(const char *name) const; + + // static std::vector get_config_docstrings(); +}; + +void create_caliper_config_manager_mod(py::module_ &caliper_config_manager_mod); + +} // namespace cali + +#endif /* CALI_INTERFACE_PYTHON_CONFIG_MANAGER_H */ \ No newline at end of file diff --git a/bindings/python/instrumentation.cpp b/bindings/python/instrumentation.cpp new file mode 100644 index 000000000..713575b31 --- /dev/null +++ b/bindings/python/instrumentation.cpp @@ -0,0 +1,119 @@ +#include "instrumentation.h" +#include + +namespace cali { + +PythonAttribute::PythonAttribute(const char *name, cali_attr_type type, + cali_attr_properties opt) + : m_attr_id(cali_create_attribute(name, type, opt)) { + if (m_attr_id == CALI_INV_ID) { + throw std::runtime_error("Failed to create attribute"); + } +} + +PythonAttribute::PythonAttribute(cali_id_t id) { + if (id == CALI_INV_ID) { + throw std::runtime_error("Invalid attribute"); + } + m_attr_id = id; +} + +PythonAttribute PythonAttribute::find_attribute(const char *name) { + PythonAttribute found_attr{cali_find_attribute(name)}; + return found_attr; +} + +const char *PythonAttribute::name() const { + return cali_attribute_name(m_attr_id); +} + +cali_attr_type PythonAttribute::type() const { + return cali_attribute_type(m_attr_id); +} + +cali_attr_properties PythonAttribute::properties() const { + return static_cast( + cali_attribute_properties(m_attr_id)); +} + +void PythonAttribute::begin() { cali_begin(m_attr_id); } + +void PythonAttribute::begin(int val) { cali_begin_int(m_attr_id, val); } + +void PythonAttribute::begin(double val) { cali_begin_double(m_attr_id, val); } + +void PythonAttribute::begin(const char *val) { + cali_begin_string(m_attr_id, val); +} + +void PythonAttribute::set(int val) { cali_set_int(m_attr_id, val); } + +void PythonAttribute::set(double val) { cali_set_double(m_attr_id, val); } + +void PythonAttribute::set(const char *val) { cali_set_string(m_attr_id, val); } + +void PythonAttribute::end() { cali_end(m_attr_id); } + +void create_caliper_instrumentation_mod( + py::module_ &caliper_instrumentation_mod) { + // PythonAttribute bindings + py::class_ cali_attribute_type(caliper_instrumentation_mod, + "Attribute"); + cali_attribute_type.def( + py::init(), "", + py::arg(), py::arg(), py::arg("opt") = 0); + cali_attribute_type.def_static("find_attribute", + &PythonAttribute::find_attribute); + cali_attribute_type.def_property_readonly("name", &PythonAttribute::name); + cali_attribute_type.def_property_readonly("type", &PythonAttribute::type); + cali_attribute_type.def_property_readonly("properties", + &PythonAttribute::properties); + cali_attribute_type.def("begin", static_cast( + &PythonAttribute::begin)); + cali_attribute_type.def("begin", static_cast( + &PythonAttribute::begin)); + cali_attribute_type.def( + "begin", + static_cast(&PythonAttribute::begin)); + cali_attribute_type.def("begin", + static_cast( + &PythonAttribute::begin)); + cali_attribute_type.def("set", static_cast( + &PythonAttribute::set)); + cali_attribute_type.def("set", static_cast( + &PythonAttribute::set)); + cali_attribute_type.def("set", + static_cast( + &PythonAttribute::set)); + cali_attribute_type.def("end", &PythonAttribute::end); + + // Bindings for region begin/end functions + caliper_instrumentation_mod.def("begin_region", &cali_begin_region); + caliper_instrumentation_mod.def("end_region", &cali_end_region); + caliper_instrumentation_mod.def("begin_phase", &cali_begin_phase); + caliper_instrumentation_mod.def("end_phase", &cali_end_phase); + caliper_instrumentation_mod.def("begin_comm_region", &cali_begin_comm_region); + caliper_instrumentation_mod.def("end_comm_region", &cali_end_comm_region); + + // Bindings for "_byname" functions + caliper_instrumentation_mod.def("begin_byname", &cali_begin_byname); + caliper_instrumentation_mod.def("begin_byname", &cali_begin_double_byname); + caliper_instrumentation_mod.def("begin_byname", &cali_begin_int_byname); + caliper_instrumentation_mod.def("begin_byname", &cali_begin_string_byname); + caliper_instrumentation_mod.def("set_byname", &cali_set_double_byname); + caliper_instrumentation_mod.def("set_byname", &cali_set_int_byname); + caliper_instrumentation_mod.def("set_byname", &cali_set_string_byname); + caliper_instrumentation_mod.def("end_byname", &cali_end_byname); + + // Bindings for global "_byname" functions + caliper_instrumentation_mod.def("set_global_byname", + &cali_set_global_double_byname); + caliper_instrumentation_mod.def("set_global_byname", + &cali_set_global_int_byname); + caliper_instrumentation_mod.def("set_global_byname", + &cali_set_global_string_byname); + caliper_instrumentation_mod.def("set_global_byname", + &cali_set_global_uint_byname); +} + +} // namespace cali \ No newline at end of file diff --git a/bindings/python/instrumentation.h b/bindings/python/instrumentation.h new file mode 100644 index 000000000..0c748b2fa --- /dev/null +++ b/bindings/python/instrumentation.h @@ -0,0 +1,50 @@ +#ifndef CALI_INTERFACE_PYTHON_INSTRUMENTATION_H +#define CALI_INTERFACE_PYTHON_INSTRUMENTATION_H + +#include "common.h" + +namespace cali { + +class PythonAttribute { +public: + PythonAttribute(const char *name, cali_attr_type type, + cali_attr_properties opt = CALI_ATTR_DEFAULT); + + static PythonAttribute find_attribute(const char *name); + + const char *name() const; + + cali_attr_type type() const; + + cali_attr_properties properties() const; + + void begin(); + + void begin(int val); + + void begin(double val); + + void begin(const char *val); + + void set(int val); + + void set(double val); + + void set(const char *val); + + void end(); + +private: + PythonAttribute(cali_id_t id); + + cali_id_t m_attr_id; +}; + +// TODO add "byname" functions to module + +void create_caliper_instrumentation_mod( + py::module_ &caliper_instrumentation_mod); + +} // namespace cali + +#endif /* CALI_INTERFACE_PYTHON_INSTRUMENTATION_H */ \ No newline at end of file diff --git a/bindings/python/loop.cpp b/bindings/python/loop.cpp new file mode 100644 index 000000000..3a25a8c28 --- /dev/null +++ b/bindings/python/loop.cpp @@ -0,0 +1,27 @@ +#include "loop.h" + +namespace cali { + +PythonLoop::PythonLoop(const char *name) { + if (cali_loop_attr_id == CALI_INV_ID) { + cali_init(); + } + cali_begin_string(cali_loop_attr_id, name); + m_iter_attr = cali_make_loop_iteration_attribute(name); +} + +void PythonLoop::start_iteration(int i) { cali_begin_int(m_iter_attr, i); } + +void PythonLoop::end_iteration() { cali_end(m_iter_attr); } + +void PythonLoop::end() { cali_end(cali_loop_attr_id); } + +void create_caliper_loop_mod(py::module_ &caliper_loop_mod) { + py::class_ loop_type(caliper_loop_mod, "Loop"); + loop_type.def(py::init()); + loop_type.def("start_iteration", &PythonLoop::start_iteration); + loop_type.def("end_iteration", &PythonLoop::end_iteration); + loop_type.def("end", &PythonLoop::end); +} + +} // namespace cali \ No newline at end of file diff --git a/bindings/python/loop.h b/bindings/python/loop.h new file mode 100644 index 000000000..eb483c073 --- /dev/null +++ b/bindings/python/loop.h @@ -0,0 +1,26 @@ +#ifndef CALI_INTERFACE_PYTHON_LOOP_H +#define CALI_INTERFACE_PYTHON_LOOP_H + +#include "common.h" + +namespace cali { + +class PythonLoop { +public: + PythonLoop(const char *name); + + void start_iteration(int i); + + void end_iteration(); + + void end(); + +private: + cali_id_t m_iter_attr; +}; + +void create_caliper_loop_mod(py::module_ &caliper_loop_mod); + +} // namespace cali + +#endif /* CALI_INTERFACE_PYTHON_LOOP_H */ \ No newline at end of file diff --git a/bindings/python/mod.cpp b/bindings/python/mod.cpp new file mode 100644 index 000000000..e62b8bab2 --- /dev/null +++ b/bindings/python/mod.cpp @@ -0,0 +1,69 @@ +#include "annotation.h" +#include "config_manager.h" +#include "instrumentation.h" +#include "loop.h" +#include "variant.h" + +// TODO add cali_init, cali_is_initialized, and cali_version + +bool pycaliper_is_initialized() { return cali_is_initialized() != 0; } + +PYBIND11_MODULE(__pycaliper_impl, m) { + m.attr("__version__") = cali_caliper_version(); + + m.def("init", &cali_init); + m.def("is_initialized", &pycaliper_is_initialized); + + auto types_mod = m.def_submodule("types"); + + py::enum_ c_attr_type(types_mod, "AttrType"); + c_attr_type.value("INV", CALI_TYPE_INV); + c_attr_type.value("USR", CALI_TYPE_USR); + c_attr_type.value("INT", CALI_TYPE_INT); + c_attr_type.value("UINT", CALI_TYPE_UINT); + c_attr_type.value("STRING", CALI_TYPE_STRING); + c_attr_type.value("ADDR", CALI_TYPE_ADDR); + c_attr_type.value("DOUBLE", CALI_TYPE_DOUBLE); + c_attr_type.value("BOOL", CALI_TYPE_BOOL); + c_attr_type.value("TYPE", CALI_TYPE_TYPE); + c_attr_type.value("PTR", CALI_TYPE_PTR); + c_attr_type.export_values(); + + py::enum_ c_attr_properties(types_mod, + "AttrProperties"); + c_attr_properties.value("DEFAULT", CALI_ATTR_DEFAULT); + c_attr_properties.value("ASVALUE", CALI_ATTR_ASVALUE); + c_attr_properties.value("NOMERGE", CALI_ATTR_NOMERGE); + c_attr_properties.value("SCOPE_PROCESS", CALI_ATTR_SCOPE_PROCESS); + c_attr_properties.value("SCOPE_THREAD", CALI_ATTR_SCOPE_THREAD); + c_attr_properties.value("SCOPE_TASK", CALI_ATTR_SCOPE_TASK); + c_attr_properties.value("SKIP_EVENTS", CALI_ATTR_SKIP_EVENTS); + c_attr_properties.value("HIDDEN", CALI_ATTR_HIDDEN); + c_attr_properties.value("NESTED", CALI_ATTR_NESTED); + c_attr_properties.value("GLOBAL", CALI_ATTR_GLOBAL); + c_attr_properties.value("UNALIGNED", CALI_ATTR_UNALIGNED); + c_attr_properties.value("AGGREGATABLE", CALI_ATTR_AGGREGATABLE); + c_attr_properties.value("LEVEL_1", CALI_ATTR_LEVEL_1); + c_attr_properties.value("LEVEL_2", CALI_ATTR_LEVEL_2); + c_attr_properties.value("LEVEL_3", CALI_ATTR_LEVEL_3); + c_attr_properties.value("LEVEL_4", CALI_ATTR_LEVEL_4); + c_attr_properties.value("LEVEL_5", CALI_ATTR_LEVEL_5); + c_attr_properties.value("LEVEL_6", CALI_ATTR_LEVEL_6); + c_attr_properties.value("LEVEL_7", CALI_ATTR_LEVEL_7); + c_attr_properties.export_values(); + + auto variant_mod = m.def_submodule("variant"); + cali::create_caliper_variant_mod(variant_mod); + + auto annotation_mod = m.def_submodule("annotation"); + cali::create_caliper_annotation_mod(annotation_mod); + + auto instrumentation_mod = m.def_submodule("instrumentation"); + cali::create_caliper_instrumentation_mod(instrumentation_mod); + + auto loop_mod = m.def_submodule("loop"); + cali::create_caliper_loop_mod(loop_mod); + + auto config_mgr_mod = m.def_submodule("config_manager"); + cali::create_caliper_config_manager_mod(config_mgr_mod); +} \ No newline at end of file diff --git a/bindings/python/pycaliper/__init__.py b/bindings/python/pycaliper/__init__.py new file mode 100644 index 000000000..3e271b2f1 --- /dev/null +++ b/bindings/python/pycaliper/__init__.py @@ -0,0 +1,15 @@ +from pycaliper.__pycaliper_impl import ( + __version__, + init, + is_initialized, +) + +import pycaliper.annotation +import pycaliper.config_manager +import pycaliper.high_level +import pycaliper.instrumentation +import pycaliper.loop +import pycaliper.types +import pycaliper.variant + +from pycaliper.high_level import annotate_function diff --git a/bindings/python/pycaliper/annotation.py b/bindings/python/pycaliper/annotation.py new file mode 100644 index 000000000..7cb65e44a --- /dev/null +++ b/bindings/python/pycaliper/annotation.py @@ -0,0 +1 @@ +from pycaliper.__pycaliper_impl.annoation import * diff --git a/bindings/python/pycaliper/config_manager.py b/bindings/python/pycaliper/config_manager.py new file mode 100644 index 000000000..c33a9a5cc --- /dev/null +++ b/bindings/python/pycaliper/config_manager.py @@ -0,0 +1 @@ +from pycaliper.__pycaliper_impl.config_manager import * diff --git a/bindings/python/pycaliper/high_level.py b/bindings/python/pycaliper/high_level.py new file mode 100644 index 000000000..235b35d57 --- /dev/null +++ b/bindings/python/pycaliper/high_level.py @@ -0,0 +1,44 @@ +from pycaliper.instrumentation import ( + begin_region, + end_region, + begin_phase, + end_phase, + begin_comm_region, + end_comm_region, +) + +import functools + + +__annotation_decorator_begin_map = { + "region": begin_region, + "phase": begin_phase, + "comm_region": begin_comm_region, +} + + +__annotation_decorator_end_map = { + "region": end_region, + "phase": end_phase, + "comm_region": end_comm_region, +} + + +def annotate_function(name=None, annotation_type="region"): + def inner_decorator(func): + real_name = name + if name is None or name == "": + real_name = func.__name__ + if annotation_type not in list(__annotation_decorator_begin_map.keys()): + raise ValueError("Invalid annotation type {}".format(annotation_type)) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + __annotation_decorator_begin_map[annotation_type](real_name) + result = func(*args, **kwargs) + __annotation_decorator_end_map[annotation_type](real_name) + return result + + return wrapper + + return inner_decorator diff --git a/bindings/python/pycaliper/instrumentation.py b/bindings/python/pycaliper/instrumentation.py new file mode 100644 index 000000000..46046ef57 --- /dev/null +++ b/bindings/python/pycaliper/instrumentation.py @@ -0,0 +1 @@ +from pycaliper.__pycaliper_impl.instrumentation import * diff --git a/bindings/python/pycaliper/loop.py b/bindings/python/pycaliper/loop.py new file mode 100644 index 000000000..7ee217d3a --- /dev/null +++ b/bindings/python/pycaliper/loop.py @@ -0,0 +1 @@ +from pycaliper.__pycaliper_impl.loop import * diff --git a/bindings/python/pycaliper/types.py b/bindings/python/pycaliper/types.py new file mode 100644 index 000000000..64b836fd6 --- /dev/null +++ b/bindings/python/pycaliper/types.py @@ -0,0 +1 @@ +from pycaliper.__pycaliper_impl.types import * diff --git a/bindings/python/pycaliper/variant.py b/bindings/python/pycaliper/variant.py new file mode 100644 index 000000000..9b59de2bb --- /dev/null +++ b/bindings/python/pycaliper/variant.py @@ -0,0 +1 @@ +from pycaliper.__pycaliper_impl.variant import * diff --git a/bindings/python/variant.cpp b/bindings/python/variant.cpp new file mode 100644 index 000000000..8bafe6581 --- /dev/null +++ b/bindings/python/variant.cpp @@ -0,0 +1,130 @@ +#include "variant.h" +#include + +namespace cali { + +PythonVariant::PythonVariant() : cali::Variant() {} + +PythonVariant::PythonVariant(bool val) : cali::Variant(val) {} + +PythonVariant::PythonVariant(int val) : cali::Variant(val) {} + +PythonVariant::PythonVariant(double val) : cali::Variant(val) {} + +PythonVariant::PythonVariant(unsigned int val) : cali::Variant(val) {} + +PythonVariant::PythonVariant(const char *val) : cali::Variant(val) {} + +PythonVariant::PythonVariant(cali_attr_type type, const std::string &data) + : cali::Variant(type, data.data(), data.size()) {} + +PythonVariant::PythonVariant(cali::Variant &&other) : cali::Variant(other) {} + +int64_t PythonVariant::to_int() const { + bool ok = true; + int64_t ret = cali::Variant::to_int64(&ok); + if (!ok) { + throw std::runtime_error("Could not convert Variant to int"); + } + return ret; +} + +double PythonVariant::to_float() const { + bool ok = true; + double ret = cali::Variant::to_double(&ok); + if (!ok) { + throw std::runtime_error("Could not convert Variant to double"); + } + return ret; +} + +cali_attr_type PythonVariant::to_attr_type() { + bool ok = true; + cali_attr_type ret = cali::Variant::to_attr_type(&ok); + if (!ok) { + throw std::runtime_error( + "Could not convert Variant to Caliper attribute type"); + } + return ret; +} + +py::bytes PythonVariant::pack() const { + char buf[30]; + size_t real_size = + cali::Variant::pack(reinterpret_cast(buf)); + std::string packed_variant{buf, real_size}; + return packed_variant; +} + +PythonVariant PythonVariant::unpack(py::bytes packed_variant) { + std::string cpp_packed_variant = packed_variant.cast(); + bool ok = true; + size_t variant_size = cpp_packed_variant.size(); + Variant unpacked_variant = cali::Variant::unpack( + reinterpret_cast(cpp_packed_variant.data()), + &variant_size, &ok); + if (!ok) { + throw std::runtime_error("Could not unpack variant"); + } + return unpacked_variant; +} + +void create_caliper_variant_mod(py::module_ &caliper_variant_mod) { + py::class_ variant_type(caliper_variant_mod, "Variant"); + variant_type.def(py::init<>(), "Create default Variant"); + variant_type.def(py::init(), "Create boolean Variant"); + variant_type.def(py::init(), "Create int Variant"); + variant_type.def(py::init(), "Create double Variant"); + variant_type.def(py::init(), "Create int Variant"); + variant_type.def(py::init(), "Create string Variant"); + variant_type.def(py::init(), + "Create custom Variant"); + variant_type.def("empty", &PythonVariant::empty, "Check if Variant is empty"); + variant_type.def("has_unmanaged_data", &PythonVariant::has_unmanaged_data, + "Check if Variant has unmanaged pointer data"); + variant_type.def_property_readonly("type", &PythonVariant::type, + "The type of the Variant"); + variant_type.def_property_readonly("size", &PythonVariant::size, + "The size of the Variant"); + variant_type.def("to_id", &PythonVariant::to_id, + "Get the Caliper ID for the Variant"); + variant_type.def( + "to_int", + [](const PythonVariant &variant) { + int64_t int_val = variant.to_int(); + return int_val; + }, + "Get the value of the Variant if it is an int"); + variant_type.def( + "to_float", + [](const PythonVariant &variant) { + double float_val = variant.to_float(); + return float_val; + }, + "Get the value of the Variant if it is a float"); + variant_type.def("to_attr_type", + static_cast( + &PythonVariant::to_attr_type), + "Get the type of the Variant"); + variant_type.def("to_string", &PythonVariant::to_string, + "Get the string value of the Variant"); + variant_type.def( + "pack", + [](const PythonVariant &variant) { + py::bytes packed_data = variant.pack(); + return packed_data; + }, + "Pack the Variant into a byte buffer"); + variant_type.def_static("unpack", &PythonVariant::unpack, + "Create a Variant from a byte buffer"); + variant_type.def("__eq__", [](const PythonVariant &v1, + const PythonVariant &v2) { return v1 == v2; }); + variant_type.def("__ne__", [](const PythonVariant &v1, + const PythonVariant &v2) { return v1 != v2; }); + variant_type.def("__lt__", [](const PythonVariant &v1, + const PythonVariant &v2) { return v1 < v2; }); + variant_type.def("__gt__", [](const PythonVariant &v1, + const PythonVariant &v2) { return v1 > v2; }); +} + +} // namespace cali \ No newline at end of file diff --git a/bindings/python/variant.h b/bindings/python/variant.h new file mode 100644 index 000000000..e1e767104 --- /dev/null +++ b/bindings/python/variant.h @@ -0,0 +1,42 @@ +#ifndef CALI_INTERFACE_PYTHON_VARIANT_H +#define CALI_INTERFACE_PYTHON_VARIANT_H + +#include "common.h" + +namespace cali { + +class PythonVariant : public cali::Variant { +public: + PythonVariant(); + + PythonVariant(bool val); + + PythonVariant(int val); + + PythonVariant(double val); + + PythonVariant(unsigned int val); + + PythonVariant(const char *val); + + PythonVariant(cali_attr_type type, const std::string &data); + + int64_t to_int() const; + + double to_float() const; + + cali_attr_type to_attr_type(); + + py::bytes pack() const; + + static PythonVariant unpack(py::bytes packed_variant); + +private: + PythonVariant(Variant &&other); +}; + +void create_caliper_variant_mod(py::module_ &caliper_variant_mod); + +} // namespace cali + +#endif /* CALI_INTERFACE_PYTHON_VARIANT_H */ \ No newline at end of file diff --git a/cmake/get_python_install_paths.py b/cmake/get_python_install_paths.py new file mode 100644 index 000000000..eb4823207 --- /dev/null +++ b/cmake/get_python_install_paths.py @@ -0,0 +1,14 @@ +import sys +import sysconfig + +if len(sys.argv) != 3 or sys.argv[1] not in ("purelib", "platlib"): + raise RuntimeError( + "Usage: python get_python_install_paths.py " + ) + +install_dir = sysconfig.get_path(sys.argv[1], sys.argv[2], {"userbase": "", "base": ""}) + +if install_dir.startswith("/"): + install_dir = install_dir[1:] + +print(install_dir, end="") From 9c8d041f26ab0dc7e0bb1672d75072de57d525d8 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Sat, 13 Jul 2024 20:47:17 -0700 Subject: [PATCH 02/12] Improves the exported type enums --- bindings/python/mod.cpp | 62 ++++++++++++------------- bindings/python/pycaliper/annotation.py | 2 +- bindings/python/pycaliper/types.py | 26 +++++++++++ 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/bindings/python/mod.cpp b/bindings/python/mod.cpp index e62b8bab2..1d68200d8 100644 --- a/bindings/python/mod.cpp +++ b/bindings/python/mod.cpp @@ -16,40 +16,40 @@ PYBIND11_MODULE(__pycaliper_impl, m) { auto types_mod = m.def_submodule("types"); - py::enum_ c_attr_type(types_mod, "AttrType"); - c_attr_type.value("INV", CALI_TYPE_INV); - c_attr_type.value("USR", CALI_TYPE_USR); - c_attr_type.value("INT", CALI_TYPE_INT); - c_attr_type.value("UINT", CALI_TYPE_UINT); - c_attr_type.value("STRING", CALI_TYPE_STRING); - c_attr_type.value("ADDR", CALI_TYPE_ADDR); - c_attr_type.value("DOUBLE", CALI_TYPE_DOUBLE); - c_attr_type.value("BOOL", CALI_TYPE_BOOL); - c_attr_type.value("TYPE", CALI_TYPE_TYPE); - c_attr_type.value("PTR", CALI_TYPE_PTR); + py::enum_ c_attr_type(types_mod, "AttrTypeEnum"); + c_attr_type.value("CALI_TYPE_INV", CALI_TYPE_INV); + c_attr_type.value("CALI_TYPE_USR", CALI_TYPE_USR); + c_attr_type.value("CALI_TYPE_INT", CALI_TYPE_INT); + c_attr_type.value("CALI_TYPE_UINT", CALI_TYPE_UINT); + c_attr_type.value("CALI_TYPE_STRING", CALI_TYPE_STRING); + c_attr_type.value("CALI_TYPE_ADDR", CALI_TYPE_ADDR); + c_attr_type.value("CALI_TYPE_DOUBLE", CALI_TYPE_DOUBLE); + c_attr_type.value("CALI_TYPE_BOOL", CALI_TYPE_BOOL); + c_attr_type.value("CALI_TYPE_TYPE", CALI_TYPE_TYPE); + c_attr_type.value("CALI_TYPE_PTR", CALI_TYPE_PTR); c_attr_type.export_values(); py::enum_ c_attr_properties(types_mod, - "AttrProperties"); - c_attr_properties.value("DEFAULT", CALI_ATTR_DEFAULT); - c_attr_properties.value("ASVALUE", CALI_ATTR_ASVALUE); - c_attr_properties.value("NOMERGE", CALI_ATTR_NOMERGE); - c_attr_properties.value("SCOPE_PROCESS", CALI_ATTR_SCOPE_PROCESS); - c_attr_properties.value("SCOPE_THREAD", CALI_ATTR_SCOPE_THREAD); - c_attr_properties.value("SCOPE_TASK", CALI_ATTR_SCOPE_TASK); - c_attr_properties.value("SKIP_EVENTS", CALI_ATTR_SKIP_EVENTS); - c_attr_properties.value("HIDDEN", CALI_ATTR_HIDDEN); - c_attr_properties.value("NESTED", CALI_ATTR_NESTED); - c_attr_properties.value("GLOBAL", CALI_ATTR_GLOBAL); - c_attr_properties.value("UNALIGNED", CALI_ATTR_UNALIGNED); - c_attr_properties.value("AGGREGATABLE", CALI_ATTR_AGGREGATABLE); - c_attr_properties.value("LEVEL_1", CALI_ATTR_LEVEL_1); - c_attr_properties.value("LEVEL_2", CALI_ATTR_LEVEL_2); - c_attr_properties.value("LEVEL_3", CALI_ATTR_LEVEL_3); - c_attr_properties.value("LEVEL_4", CALI_ATTR_LEVEL_4); - c_attr_properties.value("LEVEL_5", CALI_ATTR_LEVEL_5); - c_attr_properties.value("LEVEL_6", CALI_ATTR_LEVEL_6); - c_attr_properties.value("LEVEL_7", CALI_ATTR_LEVEL_7); + "AttrPropertiesEnum"); + c_attr_properties.value("CALI_ATTR_DEFAULT", CALI_ATTR_DEFAULT); + c_attr_properties.value("CALI_ATTR_ASVALUE", CALI_ATTR_ASVALUE); + c_attr_properties.value("CALI_ATTR_NOMERGE", CALI_ATTR_NOMERGE); + c_attr_properties.value("CALI_ATTR_SCOPE_PROCESS", CALI_ATTR_SCOPE_PROCESS); + c_attr_properties.value("CALI_ATTR_SCOPE_THREAD", CALI_ATTR_SCOPE_THREAD); + c_attr_properties.value("CALI_ATTR_SCOPE_TASK", CALI_ATTR_SCOPE_TASK); + c_attr_properties.value("CALI_ATTR_SKIP_EVENTS", CALI_ATTR_SKIP_EVENTS); + c_attr_properties.value("CALI_ATTR_HIDDEN", CALI_ATTR_HIDDEN); + c_attr_properties.value("CALI_ATTR_NESTED", CALI_ATTR_NESTED); + c_attr_properties.value("CALI_ATTR_GLOBAL", CALI_ATTR_GLOBAL); + c_attr_properties.value("CALI_ATTR_UNALIGNED", CALI_ATTR_UNALIGNED); + c_attr_properties.value("CALI_ATTR_AGGREGATABLE", CALI_ATTR_AGGREGATABLE); + c_attr_properties.value("CALI_ATTR_LEVEL_1", CALI_ATTR_LEVEL_1); + c_attr_properties.value("CALI_ATTR_LEVEL_2", CALI_ATTR_LEVEL_2); + c_attr_properties.value("CALI_ATTR_LEVEL_3", CALI_ATTR_LEVEL_3); + c_attr_properties.value("CALI_ATTR_LEVEL_4", CALI_ATTR_LEVEL_4); + c_attr_properties.value("CALI_ATTR_LEVEL_5", CALI_ATTR_LEVEL_5); + c_attr_properties.value("CALI_ATTR_LEVEL_6", CALI_ATTR_LEVEL_6); + c_attr_properties.value("CALI_ATTR_LEVEL_7", CALI_ATTR_LEVEL_7); c_attr_properties.export_values(); auto variant_mod = m.def_submodule("variant"); diff --git a/bindings/python/pycaliper/annotation.py b/bindings/python/pycaliper/annotation.py index 7cb65e44a..0b1f95104 100644 --- a/bindings/python/pycaliper/annotation.py +++ b/bindings/python/pycaliper/annotation.py @@ -1 +1 @@ -from pycaliper.__pycaliper_impl.annoation import * +from pycaliper.__pycaliper_impl.annotation import * diff --git a/bindings/python/pycaliper/types.py b/bindings/python/pycaliper/types.py index 64b836fd6..8426b863c 100644 --- a/bindings/python/pycaliper/types.py +++ b/bindings/python/pycaliper/types.py @@ -1 +1,27 @@ from pycaliper.__pycaliper_impl.types import * + +from enum import Enum + + +_CALI_TYPE_ENUM_PREFIX = "CALI_TYPE_" +_CALI_PROPERTIES_ENUM_PREFIX = "CALI_ATTR_" + + +AttrType = Enum( + "AttrType", + { + name[len(_CALI_TYPE_ENUM_PREFIX) :]: val + for name, val in AttrTypeEnum.__members__.items() + if name.startswith(_CALI_TYPE_ENUM_PREFIX) + }, +) + + +AttrProperties = Enum( + "AttrProperties", + { + name[len(_CALI_PROPERTIES_ENUM_PREFIX) :]: val + for name, val in AttrPropertiesEnum.__members__.items() + if name.startswith(_CALI_PROPERTIES_ENUM_PREFIX) + }, +) From 7f7e7e43a24653c6e0c6108ce93e658231dec57a Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 15 Jul 2024 17:33:57 -0700 Subject: [PATCH 03/12] Adds examples of the Python API --- .../apps/cali-perfproblem-branch-mispred.py | 47 ++++++++++++ examples/apps/py-example.py | 76 +++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 examples/apps/cali-perfproblem-branch-mispred.py create mode 100644 examples/apps/py-example.py diff --git a/examples/apps/cali-perfproblem-branch-mispred.py b/examples/apps/cali-perfproblem-branch-mispred.py new file mode 100644 index 000000000..a5a550abb --- /dev/null +++ b/examples/apps/cali-perfproblem-branch-mispred.py @@ -0,0 +1,47 @@ +# Copyright (c) 2024, Lawrence Livermore National Security, LLC. +# See top-level LICENSE file for details. + +from pycaliper.high_level import annotate_function +from pycaliper.annotation import Annotation + +import numpy as np + +@annotate_function() +def init(arraySize: int, sort: bool) -> np.array: + data = np.random.randint(256, size=arraySize) + if sort: + data = np.sort(data) + return data + + +@annotate_function() +def work(data: np.array): + data_sum = 0 + for _ in range(100000): + for val in np.nditer(data): + if val >= 128: + data_sum += val + print("sum =", data_sum) + + +@annotate_function() +def benchmark(arraySize: int, sort: bool): + sorted_ann = Annotation("sorted") + sorted_ann.set(sort) + print("Intializing benchmark data with sort =", sort) + data = init(arraySize, sort) + print("Calculating sum of values >= 128") + work(data) + print("Done!") + + +@annotate_function() +def main(): + arraySize = 32768 + benchmark(arraySize, True) + benchmark(arraySize, False) + + +if __name__ == "__main__": + main() + \ No newline at end of file diff --git a/examples/apps/py-example.py b/examples/apps/py-example.py new file mode 100644 index 000000000..690c7ca43 --- /dev/null +++ b/examples/apps/py-example.py @@ -0,0 +1,76 @@ +# Copyright (c) 2024, Lawrence Livermore National Security, LLC. +# See top-level LICENSE file for details. + +from pycaliper.high_level import annotate_function +from pycaliper.config_manager import ConfigManager +from pycaliper.instrumentation import ( + set_global_byname, + begin_region, + end_region, +) +from pycaliper.loop import Loop + +import argparse +import sys +import time + + +def get_available_specs_doc(mgr: ConfigManager): + doc = "" + for cfg in mgr.available_config_specs(): + doc += mgr.get_documentation_for_spec(cfg) + doc += "\n" + return doc + + +@annotate_function() +def foo(i: int) -> float: + nsecs = max(i * 500, 100000) + secs = nsecs / 10**9 + time.sleep(secs) + return 0.5 * i + + +def main(): + mgr = ConfigManager() + + parser = argparse.ArgumentParser() + parser.add_argument("--caliper_config", "-P", type=str, default="", + help="Configuration for Caliper\n{}".format(get_available_specs_doc(mgr))) + parser.add_argument("iterations", type=int, nargs="?", default=4, + help="Number of iterations") + args = parser.parse_args() + + mgr.add(args.caliper_config) + + if mgr.error(): + print("Caliper config error:", mgr, file=sys.stderr) + + mgr.start() + + set_global_byname("iterations", args.iterations) + set_global_byname("caliper.config", args.caliper_config) + + begin_region("main") + + begin_region("init") + t = 0 + end_region("init") + + loop_ann = Loop("mainloop") + + for i in range(args.iterations): + loop_ann.start_iteration(i) + t *= foo(i) + loop_ann.end_iteration() + + loop_ann.end() + + end_region("main") + + mgr.flush() + + +if __name__ == "__main__": + main() + \ No newline at end of file From c3cf13d37819572756d427a54cb6d06261467f6a Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 15 Jul 2024 17:48:10 -0700 Subject: [PATCH 04/12] Fixes default parameter for cali_attr_properties --- bindings/python/annotation.cpp | 2 +- bindings/python/instrumentation.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/annotation.cpp b/bindings/python/annotation.cpp index 8af1d7d01..0f0702edc 100644 --- a/bindings/python/annotation.cpp +++ b/bindings/python/annotation.cpp @@ -66,7 +66,7 @@ void create_caliper_annotation_mod(py::module_ &caliper_annotation_mod) { py::class_ annotation_type(caliper_annotation_mod, "Annotation"); annotation_type.def(py::init(), "", - py::arg(), py::arg("opt") = 0); + py::arg(), py::arg("opt") = CALI_ATTR_DEFAULT); annotation_type.def("end", &PythonAnnotation::end, ""); annotation_type.def("begin", static_cast( diff --git a/bindings/python/instrumentation.cpp b/bindings/python/instrumentation.cpp index 713575b31..0d63f8fc0 100644 --- a/bindings/python/instrumentation.cpp +++ b/bindings/python/instrumentation.cpp @@ -61,7 +61,7 @@ void create_caliper_instrumentation_mod( "Attribute"); cali_attribute_type.def( py::init(), "", - py::arg(), py::arg(), py::arg("opt") = 0); + py::arg(), py::arg(), py::arg("opt") = CALI_ATTR_DEFAULT); cali_attribute_type.def_static("find_attribute", &PythonAttribute::find_attribute); cali_attribute_type.def_property_readonly("name", &PythonAttribute::name); From ae50a4a4a3da606e202a010a9dbd5c13656bbc8d Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 15 Jul 2024 18:03:35 -0700 Subject: [PATCH 05/12] Removes default arugments for Attribute and Annotation in place of multiple constructors --- bindings/python/annotation.cpp | 7 ++++++- bindings/python/annotation.h | 4 +++- bindings/python/instrumentation.cpp | 12 +++++++++++- bindings/python/instrumentation.h | 4 +++- examples/apps/cali-perfproblem-branch-mispred.py | 5 +++-- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/bindings/python/annotation.cpp b/bindings/python/annotation.cpp index 0f0702edc..fababae76 100644 --- a/bindings/python/annotation.cpp +++ b/bindings/python/annotation.cpp @@ -2,6 +2,9 @@ namespace cali { +PythonAnnotation::PythonAnnotation(const char *name) + : cali::Annotation(name, CALI_ATTR_DEFAULT) {} + PythonAnnotation::PythonAnnotation(const char *name, cali_attr_properties opt) : cali::Annotation(name, opt) {} @@ -65,8 +68,10 @@ PythonAnnotation &PythonAnnotation::set(PythonVariant &data) { void create_caliper_annotation_mod(py::module_ &caliper_annotation_mod) { py::class_ annotation_type(caliper_annotation_mod, "Annotation"); + annotation_type.def(py::init(), "", + py::arg()); annotation_type.def(py::init(), "", - py::arg(), py::arg("opt") = CALI_ATTR_DEFAULT); + py::arg(), py::arg("opt")); annotation_type.def("end", &PythonAnnotation::end, ""); annotation_type.def("begin", static_cast( diff --git a/bindings/python/annotation.h b/bindings/python/annotation.h index 0c15a523f..35f2421f1 100644 --- a/bindings/python/annotation.h +++ b/bindings/python/annotation.h @@ -7,8 +7,10 @@ namespace cali { class PythonAnnotation : public cali::Annotation { public: + PythonAnnotation(const char* name); + PythonAnnotation(const char *name, - cali_attr_properties opt = CALI_ATTR_DEFAULT); + cali_attr_properties opt); PythonAnnotation &begin(); diff --git a/bindings/python/instrumentation.cpp b/bindings/python/instrumentation.cpp index 0d63f8fc0..f8d579f44 100644 --- a/bindings/python/instrumentation.cpp +++ b/bindings/python/instrumentation.cpp @@ -3,6 +3,13 @@ namespace cali { +PythonAttribute::PythonAttribute(const char *name, cali_attr_type type) + : m_attr_id(cali_create_attribute(name, type, CALI_ATTR_DEFAULT)) { + if (m_attr_id == CALI_INV_ID) { + throw std::runtime_error("Failed to create attribute"); + } +} + PythonAttribute::PythonAttribute(const char *name, cali_attr_type type, cali_attr_properties opt) : m_attr_id(cali_create_attribute(name, type, opt)) { @@ -59,9 +66,12 @@ void create_caliper_instrumentation_mod( // PythonAttribute bindings py::class_ cali_attribute_type(caliper_instrumentation_mod, "Attribute"); + cali_attribute_type.def( + py::init(), "", + py::arg(), py::arg()); cali_attribute_type.def( py::init(), "", - py::arg(), py::arg(), py::arg("opt") = CALI_ATTR_DEFAULT); + py::arg(), py::arg(), py::arg("opt")); cali_attribute_type.def_static("find_attribute", &PythonAttribute::find_attribute); cali_attribute_type.def_property_readonly("name", &PythonAttribute::name); diff --git a/bindings/python/instrumentation.h b/bindings/python/instrumentation.h index 0c748b2fa..461f1cdbe 100644 --- a/bindings/python/instrumentation.h +++ b/bindings/python/instrumentation.h @@ -7,8 +7,10 @@ namespace cali { class PythonAttribute { public: + PythonAttribute(const char *name, cali_attr_type type); + PythonAttribute(const char *name, cali_attr_type type, - cali_attr_properties opt = CALI_ATTR_DEFAULT); + cali_attr_properties opt); static PythonAttribute find_attribute(const char *name); diff --git a/examples/apps/cali-perfproblem-branch-mispred.py b/examples/apps/cali-perfproblem-branch-mispred.py index a5a550abb..3d2626e69 100644 --- a/examples/apps/cali-perfproblem-branch-mispred.py +++ b/examples/apps/cali-perfproblem-branch-mispred.py @@ -17,7 +17,7 @@ def init(arraySize: int, sort: bool) -> np.array: @annotate_function() def work(data: np.array): data_sum = 0 - for _ in range(100000): + for _ in range(100): for val in np.nditer(data): if val >= 128: data_sum += val @@ -33,6 +33,7 @@ def benchmark(arraySize: int, sort: bool): print("Calculating sum of values >= 128") work(data) print("Done!") + sorted_ann.end() @annotate_function() @@ -44,4 +45,4 @@ def main(): if __name__ == "__main__": main() - \ No newline at end of file + From 7b6fdde35326783151dfb0d12d47e8e96040156f Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 15 Jul 2024 21:59:17 -0700 Subject: [PATCH 06/12] Adds unit tests for Python bindings --- bindings/python/instrumentation.cpp | 36 ++++++- bindings/python/instrumentation.h | 7 ++ bindings/python/mod.cpp | 9 +- bindings/python/pycaliper/__init__.py | 1 + bindings/python/pycaliper/types.py | 26 ----- test/ci_app_tests/CMakeLists.txt | 25 +++++ test/ci_app_tests/ci_test_py_ann.py | 74 ++++++++++++++ test/ci_app_tests/test_python_api.py | 137 ++++++++++++++++++++++++++ 8 files changed, 284 insertions(+), 31 deletions(-) create mode 100644 test/ci_app_tests/ci_test_py_ann.py create mode 100644 test/ci_app_tests/test_python_api.py diff --git a/bindings/python/instrumentation.cpp b/bindings/python/instrumentation.cpp index f8d579f44..32298d3e4 100644 --- a/bindings/python/instrumentation.cpp +++ b/bindings/python/instrumentation.cpp @@ -1,4 +1,5 @@ #include "instrumentation.h" +#include "variant.h" #include namespace cali { @@ -18,6 +19,31 @@ PythonAttribute::PythonAttribute(const char *name, cali_attr_type type, } } +PythonAttribute::PythonAttribute(const char *name, cali_attr_type type, + cali_attr_properties opt, + std::vector &meta_attrs, + std::vector &meta_vals) { + if (meta_attrs.size() != meta_vals.size()) { + throw std::runtime_error( + "'meta_attrs' and 'meta_vals' must be same length"); + } + size_t num_meta_elems = meta_attrs.size(); + cali_id_t *meta_attr_list = new cali_id_t[num_meta_elems]; + cali_variant_t *meta_val_list = new cali_variant_t[num_meta_elems]; + for (size_t i = 0; i < num_meta_elems; i++) { + meta_attr_list[i] = meta_attrs[i].m_attr_id; + meta_val_list[i] = meta_vals[i].c_variant; + } + m_attr_id = cali_create_attribute_with_metadata( + name, type, static_cast(properties), num_meta_elems, meta_attr_list, + meta_val_list); + if (m_attr_id == CALI_INV_ID) { + throw std::runtime_error("Could not create attribute with metadata"); + } + delete[] meta_val_list; + delete[] meta_attr_list; +} + PythonAttribute::PythonAttribute(cali_id_t id) { if (id == CALI_INV_ID) { throw std::runtime_error("Invalid attribute"); @@ -66,12 +92,16 @@ void create_caliper_instrumentation_mod( // PythonAttribute bindings py::class_ cali_attribute_type(caliper_instrumentation_mod, "Attribute"); - cali_attribute_type.def( - py::init(), "", - py::arg(), py::arg()); + cali_attribute_type.def(py::init(), "", + py::arg(), py::arg()); cali_attribute_type.def( py::init(), "", py::arg(), py::arg(), py::arg("opt")); + cali_attribute_type.def( + py::init &, + std::vector &>(), + ""); cali_attribute_type.def_static("find_attribute", &PythonAttribute::find_attribute); cali_attribute_type.def_property_readonly("name", &PythonAttribute::name); diff --git a/bindings/python/instrumentation.h b/bindings/python/instrumentation.h index 461f1cdbe..00d929ec7 100644 --- a/bindings/python/instrumentation.h +++ b/bindings/python/instrumentation.h @@ -3,6 +3,8 @@ #include "common.h" +#include "variant.h" + namespace cali { class PythonAttribute { @@ -12,6 +14,11 @@ class PythonAttribute { PythonAttribute(const char *name, cali_attr_type type, cali_attr_properties opt); + PythonAttribute(const char *name, cali_attr_type type, + cali_attr_properties opt, + std::vector &meta_attrs, + std::vector &meta_vals); + static PythonAttribute find_attribute(const char *name); const char *name() const; diff --git a/bindings/python/mod.cpp b/bindings/python/mod.cpp index 1d68200d8..ec46d56f0 100644 --- a/bindings/python/mod.cpp +++ b/bindings/python/mod.cpp @@ -11,12 +11,17 @@ bool pycaliper_is_initialized() { return cali_is_initialized() != 0; } PYBIND11_MODULE(__pycaliper_impl, m) { m.attr("__version__") = cali_caliper_version(); + m.def("config_preset", [](std::map &preset_map) { + for (auto kv : preset_map) { + cali_config_preset(kv.first, kv.second); + } + }); m.def("init", &cali_init); m.def("is_initialized", &pycaliper_is_initialized); auto types_mod = m.def_submodule("types"); - py::enum_ c_attr_type(types_mod, "AttrTypeEnum"); + py::enum_ c_attr_type(types_mod, "AttrType"); c_attr_type.value("CALI_TYPE_INV", CALI_TYPE_INV); c_attr_type.value("CALI_TYPE_USR", CALI_TYPE_USR); c_attr_type.value("CALI_TYPE_INT", CALI_TYPE_INT); @@ -30,7 +35,7 @@ PYBIND11_MODULE(__pycaliper_impl, m) { c_attr_type.export_values(); py::enum_ c_attr_properties(types_mod, - "AttrPropertiesEnum"); + "AttrProperties"); c_attr_properties.value("CALI_ATTR_DEFAULT", CALI_ATTR_DEFAULT); c_attr_properties.value("CALI_ATTR_ASVALUE", CALI_ATTR_ASVALUE); c_attr_properties.value("CALI_ATTR_NOMERGE", CALI_ATTR_NOMERGE); diff --git a/bindings/python/pycaliper/__init__.py b/bindings/python/pycaliper/__init__.py index 3e271b2f1..63bf6f5ff 100644 --- a/bindings/python/pycaliper/__init__.py +++ b/bindings/python/pycaliper/__init__.py @@ -1,5 +1,6 @@ from pycaliper.__pycaliper_impl import ( __version__, + config_preset, init, is_initialized, ) diff --git a/bindings/python/pycaliper/types.py b/bindings/python/pycaliper/types.py index 8426b863c..64b836fd6 100644 --- a/bindings/python/pycaliper/types.py +++ b/bindings/python/pycaliper/types.py @@ -1,27 +1 @@ from pycaliper.__pycaliper_impl.types import * - -from enum import Enum - - -_CALI_TYPE_ENUM_PREFIX = "CALI_TYPE_" -_CALI_PROPERTIES_ENUM_PREFIX = "CALI_ATTR_" - - -AttrType = Enum( - "AttrType", - { - name[len(_CALI_TYPE_ENUM_PREFIX) :]: val - for name, val in AttrTypeEnum.__members__.items() - if name.startswith(_CALI_TYPE_ENUM_PREFIX) - }, -) - - -AttrProperties = Enum( - "AttrProperties", - { - name[len(_CALI_PROPERTIES_ENUM_PREFIX) :]: val - for name, val in AttrPropertiesEnum.__members__.items() - if name.startswith(_CALI_PROPERTIES_ENUM_PREFIX) - }, -) diff --git a/test/ci_app_tests/CMakeLists.txt b/test/ci_app_tests/CMakeLists.txt index d9d99571b..a3f821473 100644 --- a/test/ci_app_tests/CMakeLists.txt +++ b/test/ci_app_tests/CMakeLists.txt @@ -24,6 +24,9 @@ set(CALIPER_CI_MPI_TEST_APPS ci_test_mpi_channel_manager) set(CALIPER_CI_Fortran_TEST_APPS ci_test_f_ann) +set(CALIPER_CI_Python_TEST_APPS + ci_test_py_ann.py +) foreach(app ${CALIPER_CI_CXX_TEST_APPS}) add_executable(${app} ${app}.cpp) @@ -130,6 +133,28 @@ if (WITH_FORTRAN) list(APPEND PYTHON_SCRIPTS test_fortran_api.py) endif() +if (WITH_PYTHON_BINDINGS) + foreach(file ${CALIPER_CI_Python_TEST_APPS}) + add_custom_target(${file} ALL + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_SOURCE_DIR}/${file} + ${CMAKE_CURRENT_BINARY_DIR}/${file}) + endforeach() + + list(APPEND PYTHON_SCRIPTS test_python_api.py) +endif() + +if (WITH_PYTHON_BINDINGS) + foreach(file ${CALIPER_CI_Python_TEST_APPS}) + add_custom_target(${file} ALL + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_SOURCE_DIR}/${file} + ${CMAKE_CURRENT_BINARY_DIR}/${file}) + endforeach() + + list(APPEND PYTHON_SCRIPTS test_python_api.py) +endif() + set(DATA_FILES example_node_info.json) diff --git a/test/ci_app_tests/ci_test_py_ann.py b/test/ci_app_tests/ci_test_py_ann.py new file mode 100644 index 000000000..017f94b30 --- /dev/null +++ b/test/ci_app_tests/ci_test_py_ann.py @@ -0,0 +1,74 @@ +# --- Caliper continuous integration test app for Python annotation interface + +from pycaliper import config_preset +from pycaliper.instrumentation import ( + Attribute, + set_global_byname, + begin_byname, + set_byname, + end_byname, +) +from pycaliper.types import AttrProperties +from pycaliper.variant import Variant +from pycaliper.config_manager import ConfigManager + +import sys + + +def main(): + config_preset({"CALI_CHANNEL_FLUSH_ON_EXIT", "false"}) + + mgr = ConfigManager() + if len(sys.argv) > 1: + mgr.add(sys.argv[1]) + + if mgr.error(): + print("Caliper config error:", mgr.err_msg(), file=sys.stderr) + exit(-1) + + mgr.start() + + set_global_byname("global.double", 42.42) + set_global_byname("global.int", 1337) + set_global_byname("global.string", "my global string") + set_global_byname("global.uint", 42) + + iter_attr = Attribute("iteration", AttrProperties.CALI_ATTR_ASVALUE) + + begin_byname("phase", "loop") + + for i in range(4): + iter_attr.begin(i) + iter_attr.end() + + end_byname("phase") + + begin_byname("ci_test_c_ann.meta-attr") + + meta_attr = Attribute("meta-attr") + meta_val = Variant(47) + + test_attr = Attribute( + "test-attr-with-metadata", + AttrProperties.CALI_ATTR_UNLIGNED, + [meta_attr], + [meta_val], + ) + + test_attr.set("abracadabra") + + end_byname("ci_test_c_ann.meta-attr") + + begin_byname("ci_test_c_ann.setbyname") + + set_byname("attr.int", 20) + set_byname("attr.dbl", 1.25) + set_byname("attr.str", "fidibus") + + end_byname("ci_test_c_ann.setbyname") + + mgr.flush() + + +if __name__ == "__main__": + main() diff --git a/test/ci_app_tests/test_python_api.py b/test/ci_app_tests/test_python_api.py new file mode 100644 index 000000000..bbc598114 --- /dev/null +++ b/test/ci_app_tests/test_python_api.py @@ -0,0 +1,137 @@ +# Tests of the Python API + +import sys + +import unittest + +import calipertest as cat + + +class CaliperPythonAPITest(unittest.TestCase): + """Caliper Python API test cases""" + + def test_py_ann_trace(self): + target_cmd = [ + sys.executable, + "./ci_test_py_ann.py", + "event-trace,output=stdout", + ] + query_cmd = ["../../src/tools/cali-query/cali-query", "-e"] + + caliper_config = {"CALI_LOG_VERBOSITY": "0"} + + query_output = cat.run_test_with_query(target_cmd, query_cmd, caliper_config) + snapshots = cat.get_snapshots_from_text(query_output) + + self.assertTrue(len(snapshots) >= 10) + + self.assertTrue( + cat.has_snapshot_with_keys( + snapshots, {"iteration", "phase", "time.duration.ns", "global.int"} + ) + ) + self.assertTrue( + cat.has_snapshot_with_attributes( + snapshots, {"event.end#phase": "loop", "phase": "loop"} + ) + ) + self.assertTrue( + cat.has_snapshot_with_attributes( + snapshots, + {"event.end#iteration": "3", "iteration": "3", "phase": "loop"}, + ) + ) + self.assertTrue( + cat.has_snapshot_with_keys( + snapshots, + {"attr.int", "attr.dbl", "attr.str", "ci_test_c_ann.setbyname"}, + ) + ) + self.assertTrue( + cat.has_snapshot_with_attributes( + snapshots, {"attr.int": "20", "attr.str": "fidibus"} + ) + ) + self.assertTrue( + cat.has_snapshot_with_attributes( + snapshots, {"test-attr-with-metadata": "abracadabra"} + ) + ) + + def test_py_ann_globals(self): + target_cmd = [sys.executable, "./ci_test_py_ann.py"] + query_cmd = ["../../src/tools/cali-query/cali-query", "-e", "--list-globals"] + + caliper_config = { + "CALI_CONFIG_PROFILE": "serial-trace", + "CALI_RECORDER_FILENAME": "stdout", + "CALI_LOG_VERBOSITY": "0", + } + + query_output = cat.run_test_with_query(target_cmd, query_cmd, caliper_config) + snapshots = cat.get_snapshots_from_text(query_output) + + self.assertTrue(len(snapshots) == 1) + + self.assertTrue( + cat.has_snapshot_with_keys( + snapshots, + { + "global.double", + "global.string", + "global.int", + "global.uint", + "cali.caliper.version", + }, + ) + ) + self.assertTrue( + cat.has_snapshot_with_attributes( + snapshots, + { + "global.int": "1337", + "global.string": "my global string", + "global.uint": "42", + }, + ) + ) + + def test_py_ann_metadata(self): + target_cmd = [sys.executable, "./ci_test_py_ann.py"] + query_cmd = [ + "../../src/tools/cali-query/cali-query", + "-e", + "--list-attributes", + "--print-attributes", + "cali.attribute.name,cali.attribute.type,meta-attr", + ] + + caliper_config = { + "CALI_CONFIG_PROFILE": "serial-trace", + "CALI_RECORDER_FILENAME": "stdout", + "CALI_LOG_VERBOSITY": "0", + } + + query_output = cat.run_test_with_query(target_cmd, query_cmd, caliper_config) + snapshots = cat.get_snapshots_from_text(query_output) + + self.assertTrue( + cat.has_snapshot_with_attributes( + snapshots, + {"cali.attribute.name": "meta-attr", "cali.attribute.type": "int"}, + ) + ) + self.assertTrue( + cat.has_snapshot_with_attributes( + snapshots, + { + "cali.attribute.name": "test-attr-with-metadata", + "cali.attribute.type": "string", + "meta-attr": "47", + }, + ) + ) + + +if __name__ == "__main__": + unittest.main() From 79649d9bd1db83b4e5c33c7f50082e982609767a Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 9 Sep 2024 15:23:38 -0400 Subject: [PATCH 07/12] Adds docstrings to Python bindings --- bindings/python/annotation.cpp | 42 ++++-- bindings/python/config_manager.cpp | 123 ++++++++++++----- bindings/python/instrumentation.cpp | 173 ++++++++++++++++-------- bindings/python/instrumentation.h | 4 +- bindings/python/loop.cpp | 10 +- bindings/python/mod.cpp | 45 ++++-- bindings/python/pycaliper/high_level.py | 9 ++ cmake/hostconfig/github-actions.cmake | 1 + 8 files changed, 287 insertions(+), 120 deletions(-) diff --git a/bindings/python/annotation.cpp b/bindings/python/annotation.cpp index fababae76..0eaf1ead7 100644 --- a/bindings/python/annotation.cpp +++ b/bindings/python/annotation.cpp @@ -68,55 +68,73 @@ PythonAnnotation &PythonAnnotation::set(PythonVariant &data) { void create_caliper_annotation_mod(py::module_ &caliper_annotation_mod) { py::class_ annotation_type(caliper_annotation_mod, "Annotation"); - annotation_type.def(py::init(), "", + annotation_type.def(py::init(), + "Creates an annotation object to manipulate the context " + "attribute with the given name.", py::arg()); - annotation_type.def(py::init(), "", + annotation_type.def(py::init(), + "Creates an annotation object to manipulate the context " + "attribute with the given name", py::arg(), py::arg("opt")); - annotation_type.def("end", &PythonAnnotation::end, ""); + annotation_type.def( + "end", &PythonAnnotation::end, + "Close the top-most open region for the associated context attribute."); annotation_type.def("begin", static_cast( &PythonAnnotation::begin), - ""); + "Begin the region for the associated context attribute"); annotation_type.def("begin", static_cast( &PythonAnnotation::begin), - ""); + "Begin the region for the associated context attribute " + "with an integer value"); annotation_type.def( "begin", static_cast( &PythonAnnotation::begin), - ""); + "Begin the region for the associated context attribute " + "with a str/bytes value"); annotation_type.def( "begin", static_cast( &PythonAnnotation::begin), - ""); + "Begin the region for the associated context attribute " + "with a str/bytes value"); annotation_type.def( "begin", static_cast( &PythonAnnotation::begin), - ""); + "Begin the region for the associated context attribute " + "with a variant"); annotation_type.def("set", static_cast( &PythonAnnotation::set), - ""); + "Exports an entry for the associated context attribute " + "with an integer value. The top-most prior open value " + "for the attribute, if any, will be overwritten."); annotation_type.def( "set", static_cast( &PythonAnnotation::set), - ""); + "Exports an entry for the associated context attribute " + "with a str/bytes value. The top-most prior open value " + "for the attribute, if any, will be overwritten."); annotation_type.def( "set", static_cast( &PythonAnnotation::set), - ""); + "Exports an entry for the associated context attribute " + "with a str/bytes value. The top-most prior open value " + "for the attribute, if any, will be overwritten."); annotation_type.def( "set", static_cast( &PythonAnnotation::set), - ""); + "Exports an entry for the associated context attribute " + "with a variant. The top-most prior open value " + "for the attribute, if any, will be overwritten."); } } // namespace cali \ No newline at end of file diff --git a/bindings/python/config_manager.cpp b/bindings/python/config_manager.cpp index 633c57063..ad4f8a5c3 100644 --- a/bindings/python/config_manager.cpp +++ b/bindings/python/config_manager.cpp @@ -31,60 +31,111 @@ void create_caliper_config_manager_mod( py::module_ &caliper_config_manager_mod) { py::class_ config_mgr_type(caliper_config_manager_mod, "ConfigManager"); - config_mgr_type.def(py::init<>()); - config_mgr_type.def(py::init()); - config_mgr_type.def("add_config_spec", - static_cast( - &cali::ConfigManager::add_config_spec)); - config_mgr_type.def("add_config_spec", - static_cast( - &PythonConfigManager::add_config_spec)); - config_mgr_type.def("add_option_spec", - static_cast( - &cali::ConfigManager::add_option_spec)); - config_mgr_type.def("add_option_spec", - static_cast( - &PythonConfigManager::add_option_spec)); - config_mgr_type.def("add", &PythonConfigManager::py_add); - config_mgr_type.def("load", - static_cast( - &cali::ConfigManager::load)); + config_mgr_type.def(py::init<>(), "Create a ConfigManager."); + config_mgr_type.def( + py::init(), + "Create a ConfigManager with the provide configuration string."); + config_mgr_type.def( + "add_config_spec", + static_cast( + &cali::ConfigManager::add_config_spec), + "Add a custom config spec to this ConfigManager." + "" + "Adds a new Caliper configuration spec for this ConfigManager" + "using a custom ChannelController or option checking function."); + config_mgr_type.def( + "add_config_spec", + static_cast( + &PythonConfigManager::add_config_spec), + "Add a JSON config spec to this ConfigManager" + "" + "Adds a new Caliper configuration specification for this ConfigManager" + "using a basic ChannelController." + "" + "See the C++ docs for more details"); + config_mgr_type.def( + "add_option_spec", + static_cast( + &cali::ConfigManager::add_option_spec), + "Add a JSON option spec to this ConfigManager" + "" + "Allows one to define options for any config in a matching category." + "Option specifications must be added before querying or creating any" + "configurations to be effective." + "" + "See the C++ docs for more details"); + config_mgr_type.def( + "add_option_spec", + static_cast( + &PythonConfigManager::add_option_spec), + "Add a JSON option spec to this ConfigManager" + "" + "Allows one to define options for any config in a matching category." + "Option specifications must be added before querying or creating any" + "configurations to be effective." + "" + "See the C++ docs for more details"); + config_mgr_type.def("add", &PythonConfigManager::py_add, + "Parse the provided configuration string and create the " + "specified configuration channels."); + config_mgr_type.def( + "load", + static_cast( + &cali::ConfigManager::load), + "Load config and option specs from the provided filename."); config_mgr_type.def( "set_default_parameter", static_cast( - &cali::ConfigManager::set_default_parameter)); + &cali::ConfigManager::set_default_parameter), + "Pre-set a key-value pair for all configurations."); config_mgr_type.def( "set_default_parameter_for_config", static_cast( - &cali::ConfigManager::set_default_parameter_for_config)); - config_mgr_type.def("error", - static_cast( - &cali::ConfigManager::error)); + &cali::ConfigManager::set_default_parameter_for_config), + "Pre-set a key-value pair for the specified configuration."); + config_mgr_type.def( + "error", + static_cast( + &cali::ConfigManager::error), + "Returns true if there was an error while parsing configuration."); config_mgr_type.def("error_msg", static_cast( - &cali::ConfigManager::error_msg)); + &cali::ConfigManager::error_msg), + "Returns an error message if there was an error while " + "parsing configuration."); config_mgr_type.def("__repr__", static_cast( &cali::ConfigManager::error_msg)); - config_mgr_type.def("start", static_cast( - &cali::ConfigManager::start)); - config_mgr_type.def("stop", static_cast( - &cali::ConfigManager::stop)); - config_mgr_type.def("flush", static_cast( - &cali::ConfigManager::flush)); - config_mgr_type.def("check", &PythonConfigManager::check); + config_mgr_type.def( + "start", + static_cast(&cali::ConfigManager::start), + "Start all configured measurement channels, or re-start paused ones."); + config_mgr_type.def( + "stop", + static_cast(&cali::ConfigManager::stop), + "Pause all configured measurement channels."); + config_mgr_type.def( + "flush", + static_cast(&cali::ConfigManager::flush), + "Flush all configured measurement channels."); + config_mgr_type.def("check", &PythonConfigManager::check, + "Check if the given config string is valid."); config_mgr_type.def( "available_config_specs", static_cast (PythonConfigManager::*)() const>( - &cali::ConfigManager::available_config_specs)); + &cali::ConfigManager::available_config_specs), + "Return names of available config specs."); config_mgr_type.def( "get_documentation_for_spec", static_cast( - &cali::ConfigManager::get_documentation_for_spec)); - config_mgr_type.def_static("get_config_docstrings", - static_cast (*)()>( - &cali::ConfigManager::get_config_docstrings)); + &cali::ConfigManager::get_documentation_for_spec), + "Return short description for the given config spec."); + config_mgr_type.def_static( + "get_config_docstrings", + static_cast (*)()>( + &cali::ConfigManager::get_config_docstrings), + "Return descriptions for available global configs."); } } // namespace cali \ No newline at end of file diff --git a/bindings/python/instrumentation.cpp b/bindings/python/instrumentation.cpp index 32298d3e4..889d44994 100644 --- a/bindings/python/instrumentation.cpp +++ b/bindings/python/instrumentation.cpp @@ -21,8 +21,8 @@ PythonAttribute::PythonAttribute(const char *name, cali_attr_type type, PythonAttribute::PythonAttribute(const char *name, cali_attr_type type, cali_attr_properties opt, - std::vector &meta_attrs, - std::vector &meta_vals) { + std::vector &meta_attrs, + std::vector &meta_vals) { if (meta_attrs.size() != meta_vals.size()) { throw std::runtime_error( "'meta_attrs' and 'meta_vals' must be same length"); @@ -32,10 +32,10 @@ PythonAttribute::PythonAttribute(const char *name, cali_attr_type type, cali_variant_t *meta_val_list = new cali_variant_t[num_meta_elems]; for (size_t i = 0; i < num_meta_elems; i++) { meta_attr_list[i] = meta_attrs[i].m_attr_id; - meta_val_list[i] = meta_vals[i].c_variant; + meta_val_list[i] = meta_vals[i].c_variant(); } m_attr_id = cali_create_attribute_with_metadata( - name, type, static_cast(properties), num_meta_elems, meta_attr_list, + name, type, static_cast(opt), num_meta_elems, meta_attr_list, meta_val_list); if (m_attr_id == CALI_INV_ID) { throw std::runtime_error("Could not create attribute with metadata"); @@ -92,68 +92,135 @@ void create_caliper_instrumentation_mod( // PythonAttribute bindings py::class_ cali_attribute_type(caliper_instrumentation_mod, "Attribute"); - cali_attribute_type.def(py::init(), "", + cali_attribute_type.def(py::init(), + "Create Caliper Attribute with name and type.", py::arg(), py::arg()); cali_attribute_type.def( - py::init(), "", - py::arg(), py::arg(), py::arg("opt")); + py::init(), + "Create Caliper Attribute with name, type, and properties.", py::arg(), + py::arg(), py::arg("opt")); cali_attribute_type.def( py::init &, - std::vector &>(), - ""); + std::vector &, std::vector &>(), + "Create Caliper Attribute with name, type, properties, and metadata"); cali_attribute_type.def_static("find_attribute", - &PythonAttribute::find_attribute); - cali_attribute_type.def_property_readonly("name", &PythonAttribute::name); - cali_attribute_type.def_property_readonly("type", &PythonAttribute::type); - cali_attribute_type.def_property_readonly("properties", - &PythonAttribute::properties); - cali_attribute_type.def("begin", static_cast( - &PythonAttribute::begin)); - cali_attribute_type.def("begin", static_cast( - &PythonAttribute::begin)); + &PythonAttribute::find_attribute, + "Get Caliper Attribute by name."); + cali_attribute_type.def_property_readonly("name", &PythonAttribute::name, + "Name of the Caliper Attribute."); + cali_attribute_type.def_property_readonly("type", &PythonAttribute::type, + "Type of the Caliper Attribute."); + cali_attribute_type.def_property_readonly( + "properties", &PythonAttribute::properties, + "Properties of the Caliper Attribute."); cali_attribute_type.def( "begin", - static_cast(&PythonAttribute::begin)); - cali_attribute_type.def("begin", - static_cast( - &PythonAttribute::begin)); - cali_attribute_type.def("set", static_cast( - &PythonAttribute::set)); - cali_attribute_type.def("set", static_cast( - &PythonAttribute::set)); - cali_attribute_type.def("set", - static_cast( - &PythonAttribute::set)); - cali_attribute_type.def("end", &PythonAttribute::end); + static_cast(&PythonAttribute::begin), + "Begin region where the value for the Attribute is 'true' on the " + "blackboard."); + cali_attribute_type.def( + "begin", + static_cast(&PythonAttribute::begin), + "Begin integer region for attribute on the blackboard."); + cali_attribute_type.def( + "begin", + static_cast(&PythonAttribute::begin), + "Begin float region for attribute on the blackboard."); + cali_attribute_type.def( + "begin", + static_cast( + &PythonAttribute::begin), + "Begin str/bytes region for attribute on the blackboard."); + cali_attribute_type.def( + "set", static_cast(&PythonAttribute::set), + "Set integer value for attribute on the blackboard."); + cali_attribute_type.def( + "set", + static_cast(&PythonAttribute::set), + "Set double value for attribute on the blackboard."); + cali_attribute_type.def( + "set", + static_cast( + &PythonAttribute::set), + "Set str/bytes value for attribute on the blackboard."); + cali_attribute_type.def( + "end", &PythonAttribute::end, + "End innermost open region for attribute on the blackboard."); // Bindings for region begin/end functions - caliper_instrumentation_mod.def("begin_region", &cali_begin_region); - caliper_instrumentation_mod.def("end_region", &cali_end_region); - caliper_instrumentation_mod.def("begin_phase", &cali_begin_phase); - caliper_instrumentation_mod.def("end_phase", &cali_end_phase); - caliper_instrumentation_mod.def("begin_comm_region", &cali_begin_comm_region); - caliper_instrumentation_mod.def("end_comm_region", &cali_end_comm_region); + caliper_instrumentation_mod.def( + "begin_region", &cali_begin_region, + "Begin nested region by name." + "" + "Begins nested region using the built-in annotation attribute."); + caliper_instrumentation_mod.def( + "end_region", &cali_end_region, + "End nested region by name." + "" + "Ends nested region using built-in annotation attribute." + "Prints an error if the name does not match the currently open region."); + caliper_instrumentation_mod.def( + "begin_phase", &cali_begin_phase, + "Begin phase region by name." + "" + "A phase marks high-level, long(er)-running code regions. While regular " + "regions" + "use the \"region\" attribute with annotation level 0, phase regions use " + "the" + "\"phase\" attribute with annotation level 4. Otherwise, phases behave" + "identical to regular Caliper regions."); + caliper_instrumentation_mod.def("end_phase", &cali_end_phase, + "End phase region by name."); + caliper_instrumentation_mod.def( + "begin_comm_region", &cali_begin_comm_region, + "Begin communication region by name." + "" + "A communication region can be used to mark communication operations" + "(e.g., MPI calls) that belong to a single communication pattern." + "They can be used to summarize communication pattern statistics." + "Otherwise, they behave identical to regular Caliper regions."); + caliper_instrumentation_mod.def("end_comm_region", &cali_end_comm_region, + "End communication region by name."); // Bindings for "_byname" functions - caliper_instrumentation_mod.def("begin_byname", &cali_begin_byname); - caliper_instrumentation_mod.def("begin_byname", &cali_begin_double_byname); - caliper_instrumentation_mod.def("begin_byname", &cali_begin_int_byname); - caliper_instrumentation_mod.def("begin_byname", &cali_begin_string_byname); - caliper_instrumentation_mod.def("set_byname", &cali_set_double_byname); - caliper_instrumentation_mod.def("set_byname", &cali_set_int_byname); - caliper_instrumentation_mod.def("set_byname", &cali_set_string_byname); - caliper_instrumentation_mod.def("end_byname", &cali_end_byname); + caliper_instrumentation_mod.def( + "begin_byname", &cali_begin_byname, + "Same as Annotation.begin, but refers to annotation by name."); + caliper_instrumentation_mod.def( + "begin_byname", &cali_begin_double_byname, + "Same as Annotation.begin, but refers to annotation by name."); + caliper_instrumentation_mod.def( + "begin_byname", &cali_begin_int_byname, + "Same as Annotation.begin, but refers to annotation by name."); + caliper_instrumentation_mod.def( + "begin_byname", &cali_begin_string_byname, + "Same as Annotation.begin, but refers to annotation by name."); + caliper_instrumentation_mod.def( + "set_byname", &cali_set_double_byname, + "Same as Annotation.set, but refers to annotation by name."); + caliper_instrumentation_mod.def( + "set_byname", &cali_set_int_byname, + "Same as Annotation.set, but refers to annotation by name."); + caliper_instrumentation_mod.def( + "set_byname", &cali_set_string_byname, + "Same as Annotation.set, but refers to annotation by name."); + caliper_instrumentation_mod.def( + "end_byname", &cali_end_byname, + "Same as Annotation.end, but refers to annotation by name."); // Bindings for global "_byname" functions - caliper_instrumentation_mod.def("set_global_byname", - &cali_set_global_double_byname); - caliper_instrumentation_mod.def("set_global_byname", - &cali_set_global_int_byname); - caliper_instrumentation_mod.def("set_global_byname", - &cali_set_global_string_byname); - caliper_instrumentation_mod.def("set_global_byname", - &cali_set_global_uint_byname); + caliper_instrumentation_mod.def( + "set_global_byname", &cali_set_global_double_byname, + "Set a global attribute with a given name to the given value."); + caliper_instrumentation_mod.def( + "set_global_byname", &cali_set_global_int_byname, + "Set a global attribute with a given name to the given value."); + caliper_instrumentation_mod.def( + "set_global_byname", &cali_set_global_string_byname, + "Set a global attribute with a given name to the given value."); + caliper_instrumentation_mod.def( + "set_global_byname", &cali_set_global_uint_byname, + "Set a global attribute with a given name to the given value."); } } // namespace cali \ No newline at end of file diff --git a/bindings/python/instrumentation.h b/bindings/python/instrumentation.h index 00d929ec7..d4460d37c 100644 --- a/bindings/python/instrumentation.h +++ b/bindings/python/instrumentation.h @@ -16,8 +16,8 @@ class PythonAttribute { PythonAttribute(const char *name, cali_attr_type type, cali_attr_properties opt, - std::vector &meta_attrs, - std::vector &meta_vals); + std::vector &meta_attrs, + std::vector &meta_vals); static PythonAttribute find_attribute(const char *name); diff --git a/bindings/python/loop.cpp b/bindings/python/loop.cpp index 3a25a8c28..c86c74b34 100644 --- a/bindings/python/loop.cpp +++ b/bindings/python/loop.cpp @@ -18,10 +18,12 @@ void PythonLoop::end() { cali_end(cali_loop_attr_id); } void create_caliper_loop_mod(py::module_ &caliper_loop_mod) { py::class_ loop_type(caliper_loop_mod, "Loop"); - loop_type.def(py::init()); - loop_type.def("start_iteration", &PythonLoop::start_iteration); - loop_type.def("end_iteration", &PythonLoop::end_iteration); - loop_type.def("end", &PythonLoop::end); + loop_type.def(py::init(), "Create a loop annotation."); + loop_type.def("start_iteration", &PythonLoop::start_iteration, + "Start a loop iteration."); + loop_type.def("end_iteration", &PythonLoop::end_iteration, + "End a loop iteration."); + loop_type.def("end", &PythonLoop::end, "End the loop annotation."); } } // namespace cali \ No newline at end of file diff --git a/bindings/python/mod.cpp b/bindings/python/mod.cpp index ec46d56f0..42daa2a5e 100644 --- a/bindings/python/mod.cpp +++ b/bindings/python/mod.cpp @@ -11,15 +11,29 @@ bool pycaliper_is_initialized() { return cali_is_initialized() != 0; } PYBIND11_MODULE(__pycaliper_impl, m) { m.attr("__version__") = cali_caliper_version(); - m.def("config_preset", [](std::map &preset_map) { - for (auto kv : preset_map) { - cali_config_preset(kv.first, kv.second); - } - }); - m.def("init", &cali_init); - m.def("is_initialized", &pycaliper_is_initialized); + m.def( + "config_preset", + [](std::map &preset_map) { + for (auto kv : preset_map) { + cali_config_preset(kv.first, kv.second); + } + }, + "Pre-set a config entry in the default config." + "The entry can still be overwritten by environment variables."); + m.def("init", &cali_init, + "Initialize Caliper." + "Typically, it is not necessary to initialize Caliper explicitly." + "Caliper will lazily initialize itself on the first Caliper API call." + "This function is used primarily by the Caliper annotation macros," + "to ensure that Caliper's pre-defined annotation attributes are" + "initialized." + "It can also be used to avoid high initialization costs in the first" + "Caliper API call."); + m.def("is_initialized", &pycaliper_is_initialized, + "Check if Caliper is initialized on this process."); - auto types_mod = m.def_submodule("types"); + auto types_mod = m.def_submodule( + "types", "Special types used by lower-level Caliper APIs."); py::enum_ c_attr_type(types_mod, "AttrType"); c_attr_type.value("CALI_TYPE_INV", CALI_TYPE_INV); @@ -57,18 +71,23 @@ PYBIND11_MODULE(__pycaliper_impl, m) { c_attr_properties.value("CALI_ATTR_LEVEL_7", CALI_ATTR_LEVEL_7); c_attr_properties.export_values(); - auto variant_mod = m.def_submodule("variant"); + auto variant_mod = + m.def_submodule("variant", "Support for Caliper Variants."); cali::create_caliper_variant_mod(variant_mod); - auto annotation_mod = m.def_submodule("annotation"); + auto annotation_mod = + m.def_submodule("annotation", "Support for Caliper annotation APIs."); cali::create_caliper_annotation_mod(annotation_mod); - auto instrumentation_mod = m.def_submodule("instrumentation"); + auto instrumentation_mod = + m.def_submodule("instrumentation", + "Support for higher-level Caliper instrumentation APIs"); cali::create_caliper_instrumentation_mod(instrumentation_mod); - auto loop_mod = m.def_submodule("loop"); + auto loop_mod = m.def_submodule("loop", "Support for loop annotations."); cali::create_caliper_loop_mod(loop_mod); - auto config_mgr_mod = m.def_submodule("config_manager"); + auto config_mgr_mod = m.def_submodule( + "config_manager", "Support for dynamic configuration of Caliper."); cali::create_caliper_config_manager_mod(config_mgr_mod); } \ No newline at end of file diff --git a/bindings/python/pycaliper/high_level.py b/bindings/python/pycaliper/high_level.py index 235b35d57..0e5d35ca7 100644 --- a/bindings/python/pycaliper/high_level.py +++ b/bindings/python/pycaliper/high_level.py @@ -25,6 +25,15 @@ def annotate_function(name=None, annotation_type="region"): + """Decorator that automatically starts and ends a region around the decorated function. + + :param name: If provided, use as the name of the created Caliper region. + If None, the name will be derived from the name of the decorated function. + :type name: str + :param annotation_type: The type of annotation to use. Can be one of "region" (uses `begin|end_region`), + "phase" (uses `begin|end_phase`), or "comm_region` (uses `begin|end_comm_region`). + """ + def inner_decorator(func): real_name = name if name is None or name == "": diff --git a/cmake/hostconfig/github-actions.cmake b/cmake/hostconfig/github-actions.cmake index 7220a4763..cc5c035b0 100644 --- a/cmake/hostconfig/github-actions.cmake +++ b/cmake/hostconfig/github-actions.cmake @@ -12,6 +12,7 @@ set(WITH_NVPROF Off CACHE BOOL "") set(WITH_PAPI Off CACHE BOOL "") set(WITH_SAMPLER On CACHE BOOL "") set(WITH_VTUNE Off CACHE BOOL "") +set(WITH_PYTHON_BINDINGS On CACHE BOOL "") set(WITH_DOCS Off CACHE BOOL "") set(BUILD_TESTING On CACHE BOOL "") From 5b53e8a9e076f80b8800ba91eff6e26df0b2d870 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Thu, 12 Sep 2024 09:16:49 -0700 Subject: [PATCH 08/12] Moves Python bindings to src/interface and adds logic to help with unit testing --- CMakeLists.txt | 8 +++--- src/CMakeLists.txt | 4 +++ .../interface}/python/CMakeLists.txt | 21 ++++++++++++++ .../interface}/python/annotation.cpp | 0 .../interface}/python/annotation.h | 0 {bindings => src/interface}/python/common.h | 0 .../interface}/python/config_manager.cpp | 0 .../interface}/python/config_manager.h | 0 .../interface}/python/instrumentation.cpp | 0 .../interface}/python/instrumentation.h | 0 {bindings => src/interface}/python/loop.cpp | 0 {bindings => src/interface}/python/loop.h | 0 {bindings => src/interface}/python/mod.cpp | 0 .../interface}/python/pycaliper/__init__.py | 0 .../interface}/python/pycaliper/annotation.py | 0 .../python/pycaliper/config_manager.py | 0 .../interface}/python/pycaliper/high_level.py | 0 .../python/pycaliper/instrumentation.py | 0 .../interface}/python/pycaliper/loop.py | 0 .../interface}/python/pycaliper/types.py | 0 .../interface}/python/pycaliper/variant.py | 0 .../interface}/python/variant.cpp | 0 {bindings => src/interface}/python/variant.h | 0 test/ci_app_tests/CMakeLists.txt | 28 +++++++++---------- test/ci_app_tests/ci_test_py_ann.py | 17 ++++++----- 25 files changed, 52 insertions(+), 26 deletions(-) rename {bindings => src/interface}/python/CMakeLists.txt (60%) rename {bindings => src/interface}/python/annotation.cpp (100%) rename {bindings => src/interface}/python/annotation.h (100%) rename {bindings => src/interface}/python/common.h (100%) rename {bindings => src/interface}/python/config_manager.cpp (100%) rename {bindings => src/interface}/python/config_manager.h (100%) rename {bindings => src/interface}/python/instrumentation.cpp (100%) rename {bindings => src/interface}/python/instrumentation.h (100%) rename {bindings => src/interface}/python/loop.cpp (100%) rename {bindings => src/interface}/python/loop.h (100%) rename {bindings => src/interface}/python/mod.cpp (100%) rename {bindings => src/interface}/python/pycaliper/__init__.py (100%) rename {bindings => src/interface}/python/pycaliper/annotation.py (100%) rename {bindings => src/interface}/python/pycaliper/config_manager.py (100%) rename {bindings => src/interface}/python/pycaliper/high_level.py (100%) rename {bindings => src/interface}/python/pycaliper/instrumentation.py (100%) rename {bindings => src/interface}/python/pycaliper/loop.py (100%) rename {bindings => src/interface}/python/pycaliper/types.py (100%) rename {bindings => src/interface}/python/pycaliper/variant.py (100%) rename {bindings => src/interface}/python/variant.cpp (100%) rename {bindings => src/interface}/python/variant.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index f7779eb93..661a1f409 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -501,15 +501,15 @@ configure_file( include_directories(${PROJECT_BINARY_DIR}/include) include_directories(include) +if (WITH_PYTHON_BINDINGS AND BUILD_TESTING) + set(PYPATH_TESTING "" CACHE INTERNAL "") +endif() + add_subdirectory(ext) add_subdirectory(src) add_subdirectory(examples/apps EXCLUDE_FROM_ALL) -if (WITH_PYTHON_BINDINGS) - add_subdirectory(bindings/python) -endif () - if (BUILD_TESTING) add_subdirectory(test) endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ac40c0dc4..1cff50221 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,6 +58,10 @@ if (WITH_TOOLS) add_subdirectory(tools) endif() +if (WITH_PYTHON_BINDINGS) + add_subdirectory(interface/python) +endif() + install( TARGETS caliper diff --git a/bindings/python/CMakeLists.txt b/src/interface/python/CMakeLists.txt similarity index 60% rename from bindings/python/CMakeLists.txt rename to src/interface/python/CMakeLists.txt index efccb982f..a7da2d1d4 100644 --- a/bindings/python/CMakeLists.txt +++ b/src/interface/python/CMakeLists.txt @@ -27,6 +27,27 @@ target_link_libraries(__pycaliper_impl PUBLIC caliper) target_compile_features(__pycaliper_impl PUBLIC cxx_std_11) target_include_directories(__pycaliper_impl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +add_custom_target( + pycaliper_test ALL # Always build pycaliper_test + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/pycaliper + COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/pycaliper ${CMAKE_CURRENT_BINARY_DIR}/pycaliper + COMMENT "Copying pycaliper Python source to ${CMAKE_CURRENT_BINARY_DIR}/pycaliper" +) +add_dependencies(__pycaliper_impl pycaliper_test) + +if (BUILD_TESTING) + set(PYPATH_TESTING ${CMAKE_CURRENT_BINARY_DIR} CACHE INTERNAL "") + add_custom_target( + pycaliper_symlink_lib_in_build ALL + COMMAND ${CMAKE_COMMAND} -E create_symlink + $ + ${CMAKE_CURRENT_BINARY_DIR}/pycaliper/$ + COMMENT "Creating symlink between Python C module and build directory for testing" + DEPENDS __pycaliper_impl + ) + message(STATUS "Will add ${PYPATH_TESTING} to PYTHONPATH during test") +endif() + install( DIRECTORY pycaliper/ diff --git a/bindings/python/annotation.cpp b/src/interface/python/annotation.cpp similarity index 100% rename from bindings/python/annotation.cpp rename to src/interface/python/annotation.cpp diff --git a/bindings/python/annotation.h b/src/interface/python/annotation.h similarity index 100% rename from bindings/python/annotation.h rename to src/interface/python/annotation.h diff --git a/bindings/python/common.h b/src/interface/python/common.h similarity index 100% rename from bindings/python/common.h rename to src/interface/python/common.h diff --git a/bindings/python/config_manager.cpp b/src/interface/python/config_manager.cpp similarity index 100% rename from bindings/python/config_manager.cpp rename to src/interface/python/config_manager.cpp diff --git a/bindings/python/config_manager.h b/src/interface/python/config_manager.h similarity index 100% rename from bindings/python/config_manager.h rename to src/interface/python/config_manager.h diff --git a/bindings/python/instrumentation.cpp b/src/interface/python/instrumentation.cpp similarity index 100% rename from bindings/python/instrumentation.cpp rename to src/interface/python/instrumentation.cpp diff --git a/bindings/python/instrumentation.h b/src/interface/python/instrumentation.h similarity index 100% rename from bindings/python/instrumentation.h rename to src/interface/python/instrumentation.h diff --git a/bindings/python/loop.cpp b/src/interface/python/loop.cpp similarity index 100% rename from bindings/python/loop.cpp rename to src/interface/python/loop.cpp diff --git a/bindings/python/loop.h b/src/interface/python/loop.h similarity index 100% rename from bindings/python/loop.h rename to src/interface/python/loop.h diff --git a/bindings/python/mod.cpp b/src/interface/python/mod.cpp similarity index 100% rename from bindings/python/mod.cpp rename to src/interface/python/mod.cpp diff --git a/bindings/python/pycaliper/__init__.py b/src/interface/python/pycaliper/__init__.py similarity index 100% rename from bindings/python/pycaliper/__init__.py rename to src/interface/python/pycaliper/__init__.py diff --git a/bindings/python/pycaliper/annotation.py b/src/interface/python/pycaliper/annotation.py similarity index 100% rename from bindings/python/pycaliper/annotation.py rename to src/interface/python/pycaliper/annotation.py diff --git a/bindings/python/pycaliper/config_manager.py b/src/interface/python/pycaliper/config_manager.py similarity index 100% rename from bindings/python/pycaliper/config_manager.py rename to src/interface/python/pycaliper/config_manager.py diff --git a/bindings/python/pycaliper/high_level.py b/src/interface/python/pycaliper/high_level.py similarity index 100% rename from bindings/python/pycaliper/high_level.py rename to src/interface/python/pycaliper/high_level.py diff --git a/bindings/python/pycaliper/instrumentation.py b/src/interface/python/pycaliper/instrumentation.py similarity index 100% rename from bindings/python/pycaliper/instrumentation.py rename to src/interface/python/pycaliper/instrumentation.py diff --git a/bindings/python/pycaliper/loop.py b/src/interface/python/pycaliper/loop.py similarity index 100% rename from bindings/python/pycaliper/loop.py rename to src/interface/python/pycaliper/loop.py diff --git a/bindings/python/pycaliper/types.py b/src/interface/python/pycaliper/types.py similarity index 100% rename from bindings/python/pycaliper/types.py rename to src/interface/python/pycaliper/types.py diff --git a/bindings/python/pycaliper/variant.py b/src/interface/python/pycaliper/variant.py similarity index 100% rename from bindings/python/pycaliper/variant.py rename to src/interface/python/pycaliper/variant.py diff --git a/bindings/python/variant.cpp b/src/interface/python/variant.cpp similarity index 100% rename from bindings/python/variant.cpp rename to src/interface/python/variant.cpp diff --git a/bindings/python/variant.h b/src/interface/python/variant.h similarity index 100% rename from bindings/python/variant.h rename to src/interface/python/variant.h diff --git a/test/ci_app_tests/CMakeLists.txt b/test/ci_app_tests/CMakeLists.txt index a3f821473..2f2f5de3d 100644 --- a/test/ci_app_tests/CMakeLists.txt +++ b/test/ci_app_tests/CMakeLists.txt @@ -135,21 +135,15 @@ endif() if (WITH_PYTHON_BINDINGS) foreach(file ${CALIPER_CI_Python_TEST_APPS}) - add_custom_target(${file} ALL - COMMAND ${CMAKE_COMMAND} -E create_symlink - ${CMAKE_CURRENT_SOURCE_DIR}/${file} - ${CMAKE_CURRENT_BINARY_DIR}/${file}) - endforeach() - - list(APPEND PYTHON_SCRIPTS test_python_api.py) -endif() - -if (WITH_PYTHON_BINDINGS) - foreach(file ${CALIPER_CI_Python_TEST_APPS}) - add_custom_target(${file} ALL - COMMAND ${CMAKE_COMMAND} -E create_symlink - ${CMAKE_CURRENT_SOURCE_DIR}/${file} - ${CMAKE_CURRENT_BINARY_DIR}/${file}) + # add_custom_target(${file} ALL + # COMMAND ${CMAKE_COMMAND} -E create_symlink + # ${CMAKE_CURRENT_SOURCE_DIR}/${file} + # ${CMAKE_CURRENT_BINARY_DIR}/${file}) + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/${file} + ${CMAKE_CURRENT_BINARY_DIR}/${file} + @ONLY + ) endforeach() list(APPEND PYTHON_SCRIPTS test_python_api.py) @@ -166,3 +160,7 @@ foreach(file ${PYTHON_SCRIPTS} ${DATA_FILES}) endforeach() add_test(NAME CI_app_tests COMMAND ${CALI_PYTHON_EXECUTABLE} -B -m unittest discover -p "test_*.py") +# if (WITH_PYTHON_BINDINGS) +# message(STATUS "Adding ${PYPATH_TESTING} to PYTHONPATH for CI_app_tests") +# set_tests_properties(CI_app_tests PROPERTIES ENVIRONMENT "PYTHONPATH=${PYPATH_TESTING}") +# endif() diff --git a/test/ci_app_tests/ci_test_py_ann.py b/test/ci_app_tests/ci_test_py_ann.py index 017f94b30..c92ee4339 100644 --- a/test/ci_app_tests/ci_test_py_ann.py +++ b/test/ci_app_tests/ci_test_py_ann.py @@ -1,5 +1,9 @@ # --- Caliper continuous integration test app for Python annotation interface +import sys + +sys.path.insert(0, "@PYPATH_TESTING@") + from pycaliper import config_preset from pycaliper.instrumentation import ( Attribute, @@ -8,15 +12,13 @@ set_byname, end_byname, ) -from pycaliper.types import AttrProperties +from pycaliper.types import CALI_TYPE_INT, CALI_ATTR_ASVALUE, CALI_TYPE_STRING, CALI_ATTR_UNALIGNED from pycaliper.variant import Variant from pycaliper.config_manager import ConfigManager -import sys - def main(): - config_preset({"CALI_CHANNEL_FLUSH_ON_EXIT", "false"}) + config_preset({"CALI_CHANNEL_FLUSH_ON_EXIT": "false"}) mgr = ConfigManager() if len(sys.argv) > 1: @@ -33,7 +35,7 @@ def main(): set_global_byname("global.string", "my global string") set_global_byname("global.uint", 42) - iter_attr = Attribute("iteration", AttrProperties.CALI_ATTR_ASVALUE) + iter_attr = Attribute("iteration", CALI_TYPE_INT, CALI_ATTR_ASVALUE) begin_byname("phase", "loop") @@ -45,12 +47,13 @@ def main(): begin_byname("ci_test_c_ann.meta-attr") - meta_attr = Attribute("meta-attr") + meta_attr = Attribute("meta-attr", CALI_TYPE_INT) meta_val = Variant(47) test_attr = Attribute( "test-attr-with-metadata", - AttrProperties.CALI_ATTR_UNLIGNED, + CALI_TYPE_STRING, + CALI_ATTR_UNALIGNED, [meta_attr], [meta_val], ) From ef0da6bb40429abde7d5adcce412da21559a3671 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Thu, 12 Sep 2024 10:12:17 -0700 Subject: [PATCH 09/12] Updates GH Actions runner to install Pybind11 correctly --- .github/workflows/cmake.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 1cd1d3bfa..3f2bf68d0 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -20,9 +20,16 @@ jobs: steps: - uses: actions/checkout@v2 + + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' - name: Install dependencies - run: sudo apt-get install libdw-dev libunwind-dev gfortran + run: | + sudo apt-get install libdw-dev libunwind-dev gfortran python-pybind11 + python3 -m pip install pybind11 - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. From d0c65691af0acfa9889b79cd0d8871bd9f595778 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Tue, 17 Sep 2024 14:54:23 -0400 Subject: [PATCH 10/12] Removes Variant from Python bindings --- src/interface/python/CMakeLists.txt | 1 - src/interface/python/annotation.cpp | 23 ---- src/interface/python/annotation.h | 9 +- src/interface/python/instrumentation.cpp | 29 ----- src/interface/python/instrumentation.h | 5 - src/interface/python/mod.cpp | 7 -- src/interface/python/pycaliper/variant.py | 1 - src/interface/python/variant.cpp | 130 ---------------------- src/interface/python/variant.h | 42 ------- 9 files changed, 2 insertions(+), 245 deletions(-) delete mode 100644 src/interface/python/pycaliper/variant.py delete mode 100644 src/interface/python/variant.cpp delete mode 100644 src/interface/python/variant.h diff --git a/src/interface/python/CMakeLists.txt b/src/interface/python/CMakeLists.txt index a7da2d1d4..6549d5cb0 100644 --- a/src/interface/python/CMakeLists.txt +++ b/src/interface/python/CMakeLists.txt @@ -3,7 +3,6 @@ set(PYCALIPER_BINDING_SOURCES config_manager.cpp instrumentation.cpp loop.cpp - variant.cpp mod.cpp ) diff --git a/src/interface/python/annotation.cpp b/src/interface/python/annotation.cpp index 0eaf1ead7..04f620528 100644 --- a/src/interface/python/annotation.cpp +++ b/src/interface/python/annotation.cpp @@ -34,11 +34,6 @@ PythonAnnotation &PythonAnnotation::begin(cali_attr_type type, return *this; } -PythonAnnotation &PythonAnnotation::begin(PythonVariant &data) { - cali::Annotation::begin(data); - return *this; -} - PythonAnnotation &PythonAnnotation::set(int data) { cali::Annotation::set(data); return *this; @@ -60,11 +55,6 @@ PythonAnnotation &PythonAnnotation::set(cali_attr_type type, return *this; } -PythonAnnotation &PythonAnnotation::set(PythonVariant &data) { - cali::Annotation::set(data); - return *this; -} - void create_caliper_annotation_mod(py::module_ &caliper_annotation_mod) { py::class_ annotation_type(caliper_annotation_mod, "Annotation"); @@ -101,12 +91,6 @@ void create_caliper_annotation_mod(py::module_ &caliper_annotation_mod) { &PythonAnnotation::begin), "Begin the region for the associated context attribute " "with a str/bytes value"); - annotation_type.def( - "begin", - static_cast( - &PythonAnnotation::begin), - "Begin the region for the associated context attribute " - "with a variant"); annotation_type.def("set", static_cast( &PythonAnnotation::set), @@ -128,13 +112,6 @@ void create_caliper_annotation_mod(py::module_ &caliper_annotation_mod) { "Exports an entry for the associated context attribute " "with a str/bytes value. The top-most prior open value " "for the attribute, if any, will be overwritten."); - annotation_type.def( - "set", - static_cast( - &PythonAnnotation::set), - "Exports an entry for the associated context attribute " - "with a variant. The top-most prior open value " - "for the attribute, if any, will be overwritten."); } } // namespace cali \ No newline at end of file diff --git a/src/interface/python/annotation.h b/src/interface/python/annotation.h index 35f2421f1..4aed96970 100644 --- a/src/interface/python/annotation.h +++ b/src/interface/python/annotation.h @@ -7,10 +7,9 @@ namespace cali { class PythonAnnotation : public cali::Annotation { public: - PythonAnnotation(const char* name); + PythonAnnotation(const char *name); - PythonAnnotation(const char *name, - cali_attr_properties opt); + PythonAnnotation(const char *name, cali_attr_properties opt); PythonAnnotation &begin(); @@ -22,8 +21,6 @@ class PythonAnnotation : public cali::Annotation { PythonAnnotation &begin(cali_attr_type type, const std::string &data); - PythonAnnotation &begin(PythonVariant &data); - PythonAnnotation &set(int data); PythonAnnotation &set(double data); @@ -31,8 +28,6 @@ class PythonAnnotation : public cali::Annotation { PythonAnnotation &set(const char *data); PythonAnnotation &set(cali_attr_type type, const std::string &data); - - PythonAnnotation &set(PythonVariant &data); }; void create_caliper_annotation_mod(py::module_ &caliper_annotation_mod); diff --git a/src/interface/python/instrumentation.cpp b/src/interface/python/instrumentation.cpp index 889d44994..27a3d7213 100644 --- a/src/interface/python/instrumentation.cpp +++ b/src/interface/python/instrumentation.cpp @@ -19,31 +19,6 @@ PythonAttribute::PythonAttribute(const char *name, cali_attr_type type, } } -PythonAttribute::PythonAttribute(const char *name, cali_attr_type type, - cali_attr_properties opt, - std::vector &meta_attrs, - std::vector &meta_vals) { - if (meta_attrs.size() != meta_vals.size()) { - throw std::runtime_error( - "'meta_attrs' and 'meta_vals' must be same length"); - } - size_t num_meta_elems = meta_attrs.size(); - cali_id_t *meta_attr_list = new cali_id_t[num_meta_elems]; - cali_variant_t *meta_val_list = new cali_variant_t[num_meta_elems]; - for (size_t i = 0; i < num_meta_elems; i++) { - meta_attr_list[i] = meta_attrs[i].m_attr_id; - meta_val_list[i] = meta_vals[i].c_variant(); - } - m_attr_id = cali_create_attribute_with_metadata( - name, type, static_cast(opt), num_meta_elems, meta_attr_list, - meta_val_list); - if (m_attr_id == CALI_INV_ID) { - throw std::runtime_error("Could not create attribute with metadata"); - } - delete[] meta_val_list; - delete[] meta_attr_list; -} - PythonAttribute::PythonAttribute(cali_id_t id) { if (id == CALI_INV_ID) { throw std::runtime_error("Invalid attribute"); @@ -99,10 +74,6 @@ void create_caliper_instrumentation_mod( py::init(), "Create Caliper Attribute with name, type, and properties.", py::arg(), py::arg(), py::arg("opt")); - cali_attribute_type.def( - py::init &, std::vector &>(), - "Create Caliper Attribute with name, type, properties, and metadata"); cali_attribute_type.def_static("find_attribute", &PythonAttribute::find_attribute, "Get Caliper Attribute by name."); diff --git a/src/interface/python/instrumentation.h b/src/interface/python/instrumentation.h index d4460d37c..78894b7ba 100644 --- a/src/interface/python/instrumentation.h +++ b/src/interface/python/instrumentation.h @@ -14,11 +14,6 @@ class PythonAttribute { PythonAttribute(const char *name, cali_attr_type type, cali_attr_properties opt); - PythonAttribute(const char *name, cali_attr_type type, - cali_attr_properties opt, - std::vector &meta_attrs, - std::vector &meta_vals); - static PythonAttribute find_attribute(const char *name); const char *name() const; diff --git a/src/interface/python/mod.cpp b/src/interface/python/mod.cpp index 42daa2a5e..8f48335a5 100644 --- a/src/interface/python/mod.cpp +++ b/src/interface/python/mod.cpp @@ -2,9 +2,6 @@ #include "config_manager.h" #include "instrumentation.h" #include "loop.h" -#include "variant.h" - -// TODO add cali_init, cali_is_initialized, and cali_version bool pycaliper_is_initialized() { return cali_is_initialized() != 0; } @@ -71,10 +68,6 @@ PYBIND11_MODULE(__pycaliper_impl, m) { c_attr_properties.value("CALI_ATTR_LEVEL_7", CALI_ATTR_LEVEL_7); c_attr_properties.export_values(); - auto variant_mod = - m.def_submodule("variant", "Support for Caliper Variants."); - cali::create_caliper_variant_mod(variant_mod); - auto annotation_mod = m.def_submodule("annotation", "Support for Caliper annotation APIs."); cali::create_caliper_annotation_mod(annotation_mod); diff --git a/src/interface/python/pycaliper/variant.py b/src/interface/python/pycaliper/variant.py deleted file mode 100644 index 9b59de2bb..000000000 --- a/src/interface/python/pycaliper/variant.py +++ /dev/null @@ -1 +0,0 @@ -from pycaliper.__pycaliper_impl.variant import * diff --git a/src/interface/python/variant.cpp b/src/interface/python/variant.cpp deleted file mode 100644 index 8bafe6581..000000000 --- a/src/interface/python/variant.cpp +++ /dev/null @@ -1,130 +0,0 @@ -#include "variant.h" -#include - -namespace cali { - -PythonVariant::PythonVariant() : cali::Variant() {} - -PythonVariant::PythonVariant(bool val) : cali::Variant(val) {} - -PythonVariant::PythonVariant(int val) : cali::Variant(val) {} - -PythonVariant::PythonVariant(double val) : cali::Variant(val) {} - -PythonVariant::PythonVariant(unsigned int val) : cali::Variant(val) {} - -PythonVariant::PythonVariant(const char *val) : cali::Variant(val) {} - -PythonVariant::PythonVariant(cali_attr_type type, const std::string &data) - : cali::Variant(type, data.data(), data.size()) {} - -PythonVariant::PythonVariant(cali::Variant &&other) : cali::Variant(other) {} - -int64_t PythonVariant::to_int() const { - bool ok = true; - int64_t ret = cali::Variant::to_int64(&ok); - if (!ok) { - throw std::runtime_error("Could not convert Variant to int"); - } - return ret; -} - -double PythonVariant::to_float() const { - bool ok = true; - double ret = cali::Variant::to_double(&ok); - if (!ok) { - throw std::runtime_error("Could not convert Variant to double"); - } - return ret; -} - -cali_attr_type PythonVariant::to_attr_type() { - bool ok = true; - cali_attr_type ret = cali::Variant::to_attr_type(&ok); - if (!ok) { - throw std::runtime_error( - "Could not convert Variant to Caliper attribute type"); - } - return ret; -} - -py::bytes PythonVariant::pack() const { - char buf[30]; - size_t real_size = - cali::Variant::pack(reinterpret_cast(buf)); - std::string packed_variant{buf, real_size}; - return packed_variant; -} - -PythonVariant PythonVariant::unpack(py::bytes packed_variant) { - std::string cpp_packed_variant = packed_variant.cast(); - bool ok = true; - size_t variant_size = cpp_packed_variant.size(); - Variant unpacked_variant = cali::Variant::unpack( - reinterpret_cast(cpp_packed_variant.data()), - &variant_size, &ok); - if (!ok) { - throw std::runtime_error("Could not unpack variant"); - } - return unpacked_variant; -} - -void create_caliper_variant_mod(py::module_ &caliper_variant_mod) { - py::class_ variant_type(caliper_variant_mod, "Variant"); - variant_type.def(py::init<>(), "Create default Variant"); - variant_type.def(py::init(), "Create boolean Variant"); - variant_type.def(py::init(), "Create int Variant"); - variant_type.def(py::init(), "Create double Variant"); - variant_type.def(py::init(), "Create int Variant"); - variant_type.def(py::init(), "Create string Variant"); - variant_type.def(py::init(), - "Create custom Variant"); - variant_type.def("empty", &PythonVariant::empty, "Check if Variant is empty"); - variant_type.def("has_unmanaged_data", &PythonVariant::has_unmanaged_data, - "Check if Variant has unmanaged pointer data"); - variant_type.def_property_readonly("type", &PythonVariant::type, - "The type of the Variant"); - variant_type.def_property_readonly("size", &PythonVariant::size, - "The size of the Variant"); - variant_type.def("to_id", &PythonVariant::to_id, - "Get the Caliper ID for the Variant"); - variant_type.def( - "to_int", - [](const PythonVariant &variant) { - int64_t int_val = variant.to_int(); - return int_val; - }, - "Get the value of the Variant if it is an int"); - variant_type.def( - "to_float", - [](const PythonVariant &variant) { - double float_val = variant.to_float(); - return float_val; - }, - "Get the value of the Variant if it is a float"); - variant_type.def("to_attr_type", - static_cast( - &PythonVariant::to_attr_type), - "Get the type of the Variant"); - variant_type.def("to_string", &PythonVariant::to_string, - "Get the string value of the Variant"); - variant_type.def( - "pack", - [](const PythonVariant &variant) { - py::bytes packed_data = variant.pack(); - return packed_data; - }, - "Pack the Variant into a byte buffer"); - variant_type.def_static("unpack", &PythonVariant::unpack, - "Create a Variant from a byte buffer"); - variant_type.def("__eq__", [](const PythonVariant &v1, - const PythonVariant &v2) { return v1 == v2; }); - variant_type.def("__ne__", [](const PythonVariant &v1, - const PythonVariant &v2) { return v1 != v2; }); - variant_type.def("__lt__", [](const PythonVariant &v1, - const PythonVariant &v2) { return v1 < v2; }); - variant_type.def("__gt__", [](const PythonVariant &v1, - const PythonVariant &v2) { return v1 > v2; }); -} - -} // namespace cali \ No newline at end of file diff --git a/src/interface/python/variant.h b/src/interface/python/variant.h deleted file mode 100644 index e1e767104..000000000 --- a/src/interface/python/variant.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef CALI_INTERFACE_PYTHON_VARIANT_H -#define CALI_INTERFACE_PYTHON_VARIANT_H - -#include "common.h" - -namespace cali { - -class PythonVariant : public cali::Variant { -public: - PythonVariant(); - - PythonVariant(bool val); - - PythonVariant(int val); - - PythonVariant(double val); - - PythonVariant(unsigned int val); - - PythonVariant(const char *val); - - PythonVariant(cali_attr_type type, const std::string &data); - - int64_t to_int() const; - - double to_float() const; - - cali_attr_type to_attr_type(); - - py::bytes pack() const; - - static PythonVariant unpack(py::bytes packed_variant); - -private: - PythonVariant(Variant &&other); -}; - -void create_caliper_variant_mod(py::module_ &caliper_variant_mod); - -} // namespace cali - -#endif /* CALI_INTERFACE_PYTHON_VARIANT_H */ \ No newline at end of file From 28dc0b137f8a199796073dd4e9eb64e83c707fde Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Tue, 17 Sep 2024 14:55:03 -0400 Subject: [PATCH 11/12] Removes redundant install of Pybind11 with apt --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 3f2bf68d0..8a47fdb72 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -28,7 +28,7 @@ jobs: - name: Install dependencies run: | - sudo apt-get install libdw-dev libunwind-dev gfortran python-pybind11 + sudo apt-get install libdw-dev libunwind-dev gfortran python3 -m pip install pybind11 - name: Configure CMake From ebd48e00bfe32b6b6efb401e8263d32c982dce2b Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Tue, 17 Sep 2024 15:12:40 -0400 Subject: [PATCH 12/12] Enables bitwise arithmetic in bindings for cali_attr_properties --- .github/workflows/cmake.yml | 2 +- src/interface/python/annotation.h | 2 +- src/interface/python/instrumentation.cpp | 2 +- src/interface/python/instrumentation.h | 2 -- src/interface/python/mod.cpp | 4 +-- src/interface/python/pycaliper/__init__.py | 1 - test/ci_app_tests/ci_test_py_ann.py | 20 +---------- test/ci_app_tests/test_python_api.py | 41 ---------------------- 8 files changed, 6 insertions(+), 68 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 8a47fdb72..d96ef6269 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -34,7 +34,7 @@ jobs: - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -C ${{github.workspace}}/cmake/hostconfig/github-actions.cmake + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -Dpybind11_DIR=$(pybind11-config --cmakedir) -C ${{github.workspace}}/cmake/hostconfig/github-actions.cmake - name: Build # Build your program with the given configuration diff --git a/src/interface/python/annotation.h b/src/interface/python/annotation.h index 4aed96970..9e1e09557 100644 --- a/src/interface/python/annotation.h +++ b/src/interface/python/annotation.h @@ -1,7 +1,7 @@ #ifndef CALI_INTERFACE_PYTHON_ANNOTATION_H #define CALI_INTERFACE_PYTHON_ANNOTATION_H -#include "variant.h" +#include "common.h" namespace cali { diff --git a/src/interface/python/instrumentation.cpp b/src/interface/python/instrumentation.cpp index 27a3d7213..ecfdc64ad 100644 --- a/src/interface/python/instrumentation.cpp +++ b/src/interface/python/instrumentation.cpp @@ -1,5 +1,5 @@ #include "instrumentation.h" -#include "variant.h" + #include namespace cali { diff --git a/src/interface/python/instrumentation.h b/src/interface/python/instrumentation.h index 78894b7ba..461f1cdbe 100644 --- a/src/interface/python/instrumentation.h +++ b/src/interface/python/instrumentation.h @@ -3,8 +3,6 @@ #include "common.h" -#include "variant.h" - namespace cali { class PythonAttribute { diff --git a/src/interface/python/mod.cpp b/src/interface/python/mod.cpp index 8f48335a5..83f53e0f6 100644 --- a/src/interface/python/mod.cpp +++ b/src/interface/python/mod.cpp @@ -45,8 +45,8 @@ PYBIND11_MODULE(__pycaliper_impl, m) { c_attr_type.value("CALI_TYPE_PTR", CALI_TYPE_PTR); c_attr_type.export_values(); - py::enum_ c_attr_properties(types_mod, - "AttrProperties"); + py::enum_ c_attr_properties(types_mod, "AttrProperties", + py::arithmetic()); c_attr_properties.value("CALI_ATTR_DEFAULT", CALI_ATTR_DEFAULT); c_attr_properties.value("CALI_ATTR_ASVALUE", CALI_ATTR_ASVALUE); c_attr_properties.value("CALI_ATTR_NOMERGE", CALI_ATTR_NOMERGE); diff --git a/src/interface/python/pycaliper/__init__.py b/src/interface/python/pycaliper/__init__.py index 63bf6f5ff..2a4ed0a8e 100644 --- a/src/interface/python/pycaliper/__init__.py +++ b/src/interface/python/pycaliper/__init__.py @@ -11,6 +11,5 @@ import pycaliper.instrumentation import pycaliper.loop import pycaliper.types -import pycaliper.variant from pycaliper.high_level import annotate_function diff --git a/test/ci_app_tests/ci_test_py_ann.py b/test/ci_app_tests/ci_test_py_ann.py index c92ee4339..4cbe27ad9 100644 --- a/test/ci_app_tests/ci_test_py_ann.py +++ b/test/ci_app_tests/ci_test_py_ann.py @@ -12,8 +12,7 @@ set_byname, end_byname, ) -from pycaliper.types import CALI_TYPE_INT, CALI_ATTR_ASVALUE, CALI_TYPE_STRING, CALI_ATTR_UNALIGNED -from pycaliper.variant import Variant +from pycaliper.types import CALI_TYPE_INT, CALI_ATTR_ASVALUE from pycaliper.config_manager import ConfigManager @@ -45,23 +44,6 @@ def main(): end_byname("phase") - begin_byname("ci_test_c_ann.meta-attr") - - meta_attr = Attribute("meta-attr", CALI_TYPE_INT) - meta_val = Variant(47) - - test_attr = Attribute( - "test-attr-with-metadata", - CALI_TYPE_STRING, - CALI_ATTR_UNALIGNED, - [meta_attr], - [meta_val], - ) - - test_attr.set("abracadabra") - - end_byname("ci_test_c_ann.meta-attr") - begin_byname("ci_test_c_ann.setbyname") set_byname("attr.int", 20) diff --git a/test/ci_app_tests/test_python_api.py b/test/ci_app_tests/test_python_api.py index bbc598114..b28da0b90 100644 --- a/test/ci_app_tests/test_python_api.py +++ b/test/ci_app_tests/test_python_api.py @@ -52,11 +52,6 @@ def test_py_ann_trace(self): snapshots, {"attr.int": "20", "attr.str": "fidibus"} ) ) - self.assertTrue( - cat.has_snapshot_with_attributes( - snapshots, {"test-attr-with-metadata": "abracadabra"} - ) - ) def test_py_ann_globals(self): target_cmd = [sys.executable, "./ci_test_py_ann.py"] @@ -96,42 +91,6 @@ def test_py_ann_globals(self): ) ) - def test_py_ann_metadata(self): - target_cmd = [sys.executable, "./ci_test_py_ann.py"] - query_cmd = [ - "../../src/tools/cali-query/cali-query", - "-e", - "--list-attributes", - "--print-attributes", - "cali.attribute.name,cali.attribute.type,meta-attr", - ] - - caliper_config = { - "CALI_CONFIG_PROFILE": "serial-trace", - "CALI_RECORDER_FILENAME": "stdout", - "CALI_LOG_VERBOSITY": "0", - } - - query_output = cat.run_test_with_query(target_cmd, query_cmd, caliper_config) - snapshots = cat.get_snapshots_from_text(query_output) - - self.assertTrue( - cat.has_snapshot_with_attributes( - snapshots, - {"cali.attribute.name": "meta-attr", "cali.attribute.type": "int"}, - ) - ) - self.assertTrue( - cat.has_snapshot_with_attributes( - snapshots, - { - "cali.attribute.name": "test-attr-with-metadata", - "cali.attribute.type": "string", - "meta-attr": "47", - }, - ) - ) - if __name__ == "__main__": unittest.main()