diff --git a/.flake8 b/.flake8 index 40ad4a7f..5a9afd25 100644 --- a/.flake8 +++ b/.flake8 @@ -9,3 +9,5 @@ max-line-length = 88 select = C,E,F,W,B,B950 ignore = E501,W503,E203 builtins = IPython +exclude = + build diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..9ed73389 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,13 @@ +hatchet/tests export-ignore +screenshot.png export-ignore +pytest.ini export-ignore +logo.png export-ignore +logo-hex.png export-ignore +install.sh export-ignore +CHANGELOG.md export-ignore +.pre-commit-config.yaml export-ignore +.flake8 export-ignore +.coveragerc export-ignore +.codecov.yml export-ignore +docs export-ignore +.github export-ignore \ No newline at end of file diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 25201d63..204803f3 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -27,15 +27,10 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install Python3 dependencies + - name: Install Hatchet run: | python -m pip install --upgrade pip pytest - pip install -r requirements.txt - # Optional Dependency for HDF Checkpointing - pip install tables - python setup.py install - python setup.py build_ext --inplace - python -m pip list + python -m pip install .[hdf_writer] - name: Update Black if: ${{ matrix.python-version == 3.9 }} @@ -91,7 +86,8 @@ jobs: - name: Test Caliper and Timemory Support with pytest run: | - PYTHONPATH=. $(which pytest) --cov=./ --cov-report=xml + # PYTHONPATH=. $(which pytest) --cov=./ --cov-report=xml + $(which pytest) --cov=./ --cov-report=xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 diff --git a/.gitignore b/.gitignore index 7473c86b..a30bdca5 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ hatchet/vis/static/*_bundle* dist/ llnl_hatchet.egg-info/ +**/__pycache__/ diff --git a/hatchet/cython_modules/meson.build b/hatchet/cython_modules/meson.build new file mode 100644 index 00000000..9566b0e7 --- /dev/null +++ b/hatchet/cython_modules/meson.build @@ -0,0 +1,16 @@ +# Specify the names of the Cython extension modules, excluding file extensions +cython_module_names = [ + 'graphframe_modules', + 'reader_modules' +] + +# Loop over the Cython modules and build/install them to +# /hatchet/cython_modules/libs +foreach mod_name : cython_module_names + py.extension_module( + mod_name, + mod_name + '.pyx', + subdir: 'hatchet' / 'cython_modules' / 'libs', + install: true + ) +endforeach \ No newline at end of file diff --git a/hatchet/external/meson.build b/hatchet/external/meson.build new file mode 100644 index 00000000..edc44dac --- /dev/null +++ b/hatchet/external/meson.build @@ -0,0 +1,20 @@ +# Specify the pure Python files for this directory +hatchet_external_python_sources = [ + '__init__.py', + 'console.py' +] + +# Install the specified pure Python files into +# /hatchet/external +py.install_sources( + hatchet_external_python_sources, + pure: false, + subdir: 'hatchet' / 'external' +) + +# Install roundtrip as-is into +# /hatchet/external +install_subdir( + 'roundtrip', + install_dir: py_top_install_dir / 'hatchet' / 'external' +) \ No newline at end of file diff --git a/hatchet/meson.build b/hatchet/meson.build new file mode 100644 index 00000000..619b2085 --- /dev/null +++ b/hatchet/meson.build @@ -0,0 +1,26 @@ +# Specify the pure Python files for this directory +hatchet_python_sources = [ + '__init__.py', + 'frame.py', + 'graph.py', + 'graphframe.py', + 'node.py', + 'version.py' +] + +# Install the specified pure Python files into +# /hatchet +py.install_sources( + hatchet_python_sources, + pure: false, + subdir: 'hatchet' +) + +# Go through all the subdirectories except for 'test' +subdir('cython_modules') +subdir('external') +subdir('query') +subdir('readers') +subdir('util') +subdir('vis') +subdir('writers') \ No newline at end of file diff --git a/hatchet/query/meson.build b/hatchet/query/meson.build new file mode 100644 index 00000000..ca478386 --- /dev/null +++ b/hatchet/query/meson.build @@ -0,0 +1,19 @@ +# Specify the pure Python files for this directory +hatchet_query_python_sources = [ + '__init__.py', + 'compat.py', + 'compound.py', + 'engine.py', + 'errors.py', + 'object_dialect.py', + 'query.py', + 'string_dialect.py' +] + +# Install the specified pure Python files into +# /hatchet/query +py.install_sources( + hatchet_query_python_sources, + pure: false, + subdir: 'hatchet' / 'query' +) \ No newline at end of file diff --git a/hatchet/readers/meson.build b/hatchet/readers/meson.build new file mode 100644 index 00000000..d6bbe922 --- /dev/null +++ b/hatchet/readers/meson.build @@ -0,0 +1,25 @@ +# Specify the pure Python files for this directory +hatchet_readers_python_sources = [ + '__init__.py', + 'caliper_native_reader.py', + 'caliper_reader.py', + 'cprofile_reader.py', + 'dataframe_reader.py', + 'gprof_dot_reader.py', + 'hdf5_reader.py', + 'hpctoolkit_reader.py', + 'json_reader.py', + 'literal_reader.py', + 'pyinstrument_reader.py', + 'spotdb_reader.py', + 'tau_reader.py', + 'timemory_reader.py' +] + +# Install the specified pure Python files into +# /hatchet/readers +py.install_sources( + hatchet_readers_python_sources, + pure: false, + subdir: 'hatchet' / 'readers' +) \ No newline at end of file diff --git a/hatchet/tests/query_compat.py b/hatchet/tests/query_compat.py index fae9b5a8..3f669c43 100644 --- a/hatchet/tests/query_compat.py +++ b/hatchet/tests/query_compat.py @@ -1,4 +1,4 @@ -# Copyright 2017-2022 Lawrence Livermore National Security, LLC and other +# Copyright 2017-2023 Lawrence Livermore National Security, LLC and other # Hatchet Project Developers. See the top-level LICENSE file for details. # # SPDX-License-Identifier: MIT diff --git a/hatchet/util/meson.build b/hatchet/util/meson.build new file mode 100644 index 00000000..b42b2f3c --- /dev/null +++ b/hatchet/util/meson.build @@ -0,0 +1,21 @@ +# Specify the pure Python files for this directory +# Note: don't include 'print_version.py' because +# that file is not needed in binary distributions +hatchet_util_python_sources = [ + '__init__.py', + 'colormaps.py', + 'config.py', + 'deprecated.py', + 'dot.py', + 'executable.py', + 'profiler.py', + 'timer.py' +] + +# Install the specified pure Python files into +# /hatchet/util +py.install_sources( + hatchet_util_python_sources, + pure: false, + subdir: 'hatchet' / 'util' +) \ No newline at end of file diff --git a/hatchet/util/print_version.py b/hatchet/util/print_version.py new file mode 100755 index 00000000..a91e5bb9 --- /dev/null +++ b/hatchet/util/print_version.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +# Copyright 2017-2023 Lawrence Livermore National Security, LLC and other +# Hatchet Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: MIT + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from version import __version__ # noqa: E402 + +print(__version__) diff --git a/hatchet/vis/meson.build b/hatchet/vis/meson.build new file mode 100644 index 00000000..595ed2d2 --- /dev/null +++ b/hatchet/vis/meson.build @@ -0,0 +1,47 @@ +# Specify the pure Python files for this directory +hatchet_vis_python_sources = [ + '__init__.py', + 'loader.py', + 'static_fixer.py' +] + +# Install the specified pure Python files into +# /hatchet/vis +py.install_sources( + hatchet_vis_python_sources, + pure: false, + subdir: 'hatchet' / 'vis' +) + +# Install directory for this source directory +hatchet_vis_install_dir = py_top_install_dir / 'hatchet' / 'vis' + +# Install all the subdirectories for visualization code as-is +# into hatchet_vis_install_dir +install_subdir( + 'external', + install_dir: hatchet_vis_install_dir +) +install_subdir( + 'scripts', + install_dir: hatchet_vis_install_dir +) +install_subdir( + 'static', + install_dir: hatchet_vis_install_dir +) +install_subdir( + 'styles', + install_dir: hatchet_vis_install_dir +) +install_subdir( + 'templates', + install_dir: hatchet_vis_install_dir +) + +# Install the following non-Python files as-is +# into hatchet_vis_install_dir +install_data( + 'package.json', 'README.md', 'requirements.txt', 'webpack.config.js', + install_dir: hatchet_vis_install_dir +) \ No newline at end of file diff --git a/hatchet/writers/meson.build b/hatchet/writers/meson.build new file mode 100644 index 00000000..da9aaabc --- /dev/null +++ b/hatchet/writers/meson.build @@ -0,0 +1,14 @@ +# Specify the pure Python files for this directory +hatchet_writers_python_sources = [ + '__init__.py', + 'dataframe_writer.py', + 'hdf5_writer.py' +] + +# Install the specified pure Python files into +# /hatchet/writers +py.install_sources( + hatchet_writers_python_sources, + pure: false, + subdir: 'hatchet' / 'writers' +) \ No newline at end of file diff --git a/install.sh b/install.sh deleted file mode 100755 index a4e0d1c1..00000000 --- a/install.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -case *"$PWD"* in - "$PYTHONPATH") - ;; - - *) - PYTHONPATH=$PWD:$PYTHONPATH - ;; -esac -python setup.py clean --all -python setup.py build_ext --inplace -python hatchet/vis/static_fixer.py diff --git a/install_editable.sh b/install_editable.sh new file mode 100644 index 00000000..c3d1f21a --- /dev/null +++ b/install_editable.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +python3 -m pip install meson[ninja] meson-python Cython + +python3 -m pip install --no-build-isolation -e . \ No newline at end of file diff --git a/meson.build b/meson.build new file mode 100644 index 00000000..be5f01bc --- /dev/null +++ b/meson.build @@ -0,0 +1,23 @@ +# Setup the Meson Project +project('llnl-hatchet', 'cython', + version: '2024.1.1' +) + +# Get the Meson Python object, a dependency object to Python for extension modules, +# and the path to the top-level install directory +py = import('python').find_installation(required: true, pure: false) +py_dep = py.dependency(method: 'system') +py_top_install_dir = py.get_install_dir() + +# Verify that the version from version.py matches the Meson project version +version_run = run_command(py.path(), meson.current_source_dir() / 'hatchet' / 'util' / 'print_version.py', check: true) +if version_run.returncode() != 0 + error('The __version__ variable in Hatchet cannot be determined') +endif +version_from_py = version_run.stdout().strip() +if not version_from_py.version_compare(meson.project_version()) + error('The __version__ variable in Hatchet does not match the Meson project version') +endif + +# Enter the 'hatchet' subdirectory +subdir('hatchet') \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 4dec4d54..5b530c92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,33 +1,65 @@ [build-system] -requires = ["setuptools", "wheel", "Cython"] -build-backend = "setuptools.build_meta" +requires = ["meson-python", "meson[ninja]", "Cython"] +build-backend = "mesonpy" -[tool.poetry] +[project] name = "llnl-hatchet" -version = "2024.1.1" -description = "A Python library for analyzing hierarchical performance data." +description = "A Python library for analyzing hierarchical performance data" +dynamic = ["version"] +readme = "./README.md" +license = { file="LICENSE" } +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: MIT License", +] authors = [ - "Abhinav Bhatele ", - "Stephanie Brink ", - "Todd Gamblin ", + { name = "Abhinav Bhatele", email = "bhatele@cs.umd.edu" }, + { name = "Stephanie Brink", email = "brink2@llnl.gov" }, + { name = "Todd Gamblin", email = "tgamblin@llnl.gov" } +] +maintainers = [ + { name = "Olga Pearce", email = "pearce8@llnl.gov" }, + { name = "Ian Lumsden", email = "ilumsden@vols.utk.edu" }, + { name = "Connor Scully-Allison", email = "cscullyallison@sci.utah.edu" }, + { name = "Dewi Yokelson", email = "dewiy@cs.uoregon.edu" }, + { name = "Michael McKinsey", email = "michaelmckinsey@tamu.edu" } +] +requires-python = ">= 3.7" +dependencies = [ + "pydot", + "PyYAML", + "matplotlib", + "numpy", + "pandas", + "textX >= 3.0.0", + "multiprocess", + "caliper-reader", +] + +[project.urls] +source_code = "https://github.com/llnl/hatchet" +documentation = "https://llnl-hatchet.readthedocs.io/en/latest/" + +[project.optional-dependencies] +hdf_writer = [ + "tables" ] -license = "MIT" [tool.ruff] line-length = 88 target-version = 'py37' include = ['\.pyi?$'] exclude = [ - ".eggs", - ".git", - ".hg", - ".mypy_cache", - ".tox", - ".venv", - "_build", - "buck-out", - "build", - "dist", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".tox", + ".venv", + "_build", + "buck-out", + "build", + "dist", ] [tool.black] diff --git a/setup.py b/setup.py deleted file mode 100644 index c2784924..00000000 --- a/setup.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2017-2023 Lawrence Livermore National Security, LLC and other -# Hatchet Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: MIT - -from setuptools import setup -from setuptools import Extension -from codecs import open -from os import path - -here = path.abspath(path.dirname(__file__)) - -# Get the long description from the README file -with open(path.join(here, "README.md"), encoding="utf-8") as f: - long_description = f.read() - -# Get the version in a safe way which does not refrence hatchet `__init__` file -# per python docs: https://packaging.python.org/guides/single-sourcing-package-version/ -version = {} -with open("./hatchet/version.py") as fp: - exec(fp.read(), version) - - -cmd_class = {} -ext_modules = [] -should_cythonize = False -mod_import_path = "hatchet.cython_modules.libs" -mod_file_path = "hatchet/cython_modules" -mod_names = [ - "reader_modules", - "graphframe_modules", -] - - -for mname in mod_names: - c_file = path.join(mod_file_path, "{}.c".format(mname)) - pyx_file = path.join(mod_file_path, "{}.pyx".format(mname)) - if not path.isfile(pyx_file): - raise FileNotFoundError( - "Requested Cython extension not found: {}".format(pyx_file) - ) - if path.isfile(c_file): - should_cythonize = False - ext_modules.append( - Extension( - "{}.{}".format(mod_import_path, mname), - [c_file], - ) - ) - else: - should_cythonize = True - ext_modules.append( - Extension( - "{}.{}".format(mod_import_path, mname), - [pyx_file], - ) - ) - -if should_cythonize: - from Cython.Build import cythonize, build_ext - - ext_modules = cythonize(ext_modules) - cmd_class.update({"build_ext": build_ext}) - - -ext_modules = [ - Extension( - "hatchet.cython_modules.libs.reader_modules", - ["hatchet/cython_modules/reader_modules.pyx"], - ), - Extension( - "hatchet.cython_modules.libs.graphframe_modules", - ["hatchet/cython_modules/graphframe_modules.pyx"], - ), -] - - -setup( - name="llnl-hatchet", - version=version["__version__"], - license="MIT", - description="A Python library for analyzing hierarchical performance data", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/llnl/hatchet", - project_urls={ - "Source Code": "https://github.com/llnl/hatchet", - "Documentation": "https://llnl-hatchet.readthedocs.io/en/latest/", - }, - author="Stephanie Brink", - author_email="brink2@llnl.gov", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: MIT License", - ], - keywords="", - python_requires=">=3.5", - packages=[ - "hatchet", - "hatchet.readers", - "hatchet.writers", - "hatchet.query", - "hatchet.vis", - "hatchet.util", - "hatchet.external", - "hatchet.tests", - "hatchet.cython_modules.libs", - ], - include_package_data=True, - install_requires=[ - "pydot", - "PyYAML", - "matplotlib", - "numpy", - "pandas", - "textX < 3.0.0; python_version < '3.6'", - "textX >= 3.0.0; python_version >= '3.6'", - "multiprocess", - "caliper-reader", - ], - ext_modules=ext_modules, - cmdclass=cmd_class, -)