From d4c1d3fcdbdb4e6b84825441aba9b73c21d9a436 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Thu, 12 Dec 2024 16:24:07 +0100 Subject: [PATCH 1/8] Switch eccodes python to wheelchains --- .github/workflows/build-wheel-linux.yml | 53 +++++------ MANIFEST.in | 20 ----- buildconfig | 14 +++ gribapi/bindings.py | 31 +++---- setup.cfg | 20 +++++ setup.py | 111 +++++++----------------- 6 files changed, 104 insertions(+), 145 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 buildconfig diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index afd539c7..3efd4199 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -7,7 +7,7 @@ # nor does it submit to any jurisdiction. -name: Build Linux +name: Build Python Wheel for Linux on: # Trigger the workflow manually @@ -16,31 +16,13 @@ on: # Allow to be called from another workflow workflow_call: ~ + # TODO automation trigger # repository_dispatch: # types: [eccodes-updated] - push: - tags-ignore: - - '**' - paths: - - 'scripts/common.sh' - - 'scripts/select-python-linux.sh' - - 'scripts/wheel-linux.sh' - - 'scripts/build-linux.sh' - - 'scripts/test-linux.sh' - - 'scripts/copy-licences.py' - - '.github/workflows/build-wheel-linux.yml' - -# to allow the action to run on the manylinux docker image based on CentOS 7 -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - jobs: - - build: - - # if: false # for temporarily disabling for debugging - + build-python-wheel: + name: Build manylinux_2_28 runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] container: image: dockcross/manylinux_2_28-x64:20250109-7bf589c @@ -152,11 +134,30 @@ jobs: name: wheel-manylinux2014-${{ matrix.python-version }} path: artifact-${{ matrix.python-version }} + build-wrapper-wheel: + name: Build manylinux_2_28 + runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] + container: + image: eccr.ecmwf.int/wheelmaker/2_28:latest + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} + steps: + # TODO convert this to be matrix-friendly. Note it's a bit tricky since + # we'd ideally not reexecute the compile step multiple times, but it + # (non-essentially) depends on a matrix-based step + # NOTE we dont use action checkout because it doesnt cleanup after itself correctly + - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj + - run: cd /proj && /buildscripts/prepare_deps.sh ./buildconfig 3.11 - run: | - source ./scripts/select-python-linux.sh 3.10 - pip3 install twine - ls -l artifact-${{ matrix.python-version }}/*.whl - twine upload artifact-${{ matrix.python-version }}/*.whl + cd /proj + export PYTHONPATH=/buildscripts/; export LIBDIR=/tmp/prereqs/eccodeslib/lib64; export INCDIR=/tmp/prereqs/eccodeslib/include + uv run --python python3.11 python -m build --installer uv --wheel . + - run: mkdir -p /build/wheel && mv /proj/dist/*whl /build/wheel + - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /build/wheel/*whl + - run: cd /proj && /buildscripts/upload-pypi.sh /build/wheel/*whl env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # NOTE temporary thing until all the mess gets cleared + - run: rm -rf ./* ./.git ./.github diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index ebd3b9e2..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,20 +0,0 @@ -include .dockerignore -include *.rst -include *.yml -include Dockerfile -include LICENSE -include Makefile -include tox.ini -include *.py -recursive-include ci *.in -recursive-include ci *.txt -recursive-include ci *.yml -recursive-include ci *.ps1 -recursive-include docs *.gitkeep -recursive-include docs *.py -recursive-include docs *.rst -recursive-include gribapi *.h -recursive-include tests *.grib2 -recursive-include tests *.grib -recursive-include tests *.ipynb -recursive-include tests *.py diff --git a/buildconfig b/buildconfig new file mode 100644 index 00000000..7ad99f6c --- /dev/null +++ b/buildconfig @@ -0,0 +1,14 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# NOTE since eccodes-python uses only parts of the Wheelmaker tooling, this file +# is only correspondingly partial. Namely, `compile` and `wheel-linux` options +# are not present, and only `prepare_deps` and `test-wheel` invocation is assumed + +DEPENDENCIES='["eccodeslib"]' +NAME="eccodes" diff --git a/gribapi/bindings.py b/gribapi/bindings.py index 690be77f..3354cc20 100644 --- a/gribapi/bindings.py +++ b/gribapi/bindings.py @@ -27,11 +27,6 @@ LOG = logging.getLogger(__name__) -_MAP = { - "grib_api": "eccodes", - "gribapi": "eccodes", -} - EXTENSIONS = { "darwin": ".dylib", "win32": ".dll", @@ -43,16 +38,13 @@ LOG.addHandler(logging.StreamHandler()) -def _lookup(name): - return _MAP.get(name, name) - - -def find_binary_libs(name): - name = _lookup(name) +def _find_eccodes_custom() -> str|None: + # TODO deprecate this method in favour of findlibs only + name = "eccodes" env_var = "ECCODES_PYTHON_USE_FINDLIBS" if int(os.environ.get(env_var, "0")): LOG.debug(f"{name} lib search: {env_var} set, so using findlibs") - + return None else: LOG.debug(f"{name} lib search: trying to find binary wheel") here = os.path.dirname(__file__) @@ -90,17 +82,14 @@ def find_binary_libs(name): LOG.debug( f"{name} lib search: did not find library from wheel; try to find as separate lib" ) + return None - # if did not find the binary wheel, or the env var is set, fall back to findlibs - import findlibs - - foundlib = findlibs.find(name) - LOG.debug(f"{name} lib search: findlibs returned {foundlib}") - return foundlib - - -library_path = find_binary_libs("eccodes") +library_path = _find_eccodes_custom() +if library_path is None: + import findlibs + library_path = findlibs.find("eccodes") + LOG.debug(f"eccodes lib search: findlibs returned {library_path}") if library_path is None: raise RuntimeError("Cannot find the ecCodes library") diff --git a/setup.cfg b/setup.cfg index 2bd557ea..c8cf29bd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,23 @@ +[metadata] +description = "eccodes" +long_description = file: README.rst +long_description_content_type = text/rst +author = "European Centre for Medium-Range Weather Forecasts (ECMWF)" +author_email = "software.support@ecmwf.int" +url = "https://github.com/ecmwf/eccodes-python" +keywords = ecCodes, GRIB, BUFR +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy + Operating System :: OS Independent + [aliases] test = pytest diff --git a/setup.py b/setup.py index 453f9c76..2dce59b8 100644 --- a/setup.py +++ b/setup.py @@ -1,106 +1,61 @@ -#!/usr/bin/env python -# -# (C) Copyright 2017- ECMWF. +# (C) Copyright 2024- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# # In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation nor -# does it submit to any jurisdiction. -# +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# TODO ideally merge this with wheelmaker's setup_utils somehow import io import os import re import sys - import setuptools +from setup_utils import parse_dependencies, ext_kwargs -def read(path): - file_path = os.path.join(os.path.dirname(__file__), *path.split("/")) - return io.open(file_path, encoding="utf-8").read() +if sys.version_info < (3, 7): + install_requires = ["numpy<1.20"] +elif sys.version_info < (3, 8): + install_requires = ["numpy<1.22"] +elif sys.version_info < (3, 9): + install_requires = ["numpy<1.25"] +else: + install_requires = ["numpy"] +install_requires += ["attrs", "cffi", "findlibs"] +ext_modules = [ + setuptools.Extension( + "eccodes._eccodes", + sources=["eccodes/_eccodes.cc"], + language="c++", + libraries=["eccodes"], + library_dirs=[os.environ["LIBDIR"]], + include_dirs=[os.environ["INCDIR"]], + ) +] -# single-sourcing the package version using method 1 of: -# https://packaging.python.org/guides/single-sourcing-package-version/ -def parse_version_from(path): +def get_version() -> str: version_pattern = ( r"^__version__ = [\"\'](.*)[\"\']" # More permissive regex pattern ) - version_file = read(path) + file_path = os.path.join(os.path.dirname(__file__), "gribapi", "bindings.py") + version_file = io.open(file_path, encoding="utf-8").read() version_match = re.search(version_pattern, version_file, re.M) if version_match is None or len(version_match.groups()) > 1: raise ValueError("couldn't parse version") return version_match.group(1) - -# for the binary wheel -libdir = os.path.realpath("install/lib") -incdir = os.path.realpath("install/include") -libs = ["eccodes"] - -if "--binary-wheel" in sys.argv: - sys.argv.remove("--binary-wheel") - - # https://setuptools.pypa.io/en/latest/userguide/ext_modules.html - ext_modules = [ - setuptools.Extension( - "eccodes._eccodes", - sources=["eccodes/_eccodes.cc"], - language="c++", - libraries=libs, - library_dirs=[libdir], - include_dirs=[incdir], - extra_link_args=["-Wl,-rpath," + libdir], - ) - ] - - def shared(directory): - result = [] - for path, dirs, files in os.walk(directory): - for f in files: - result.append(os.path.join(path, f)) - return result - - # Paths must be relative to package directory... - shared_files = ["versions.txt"] - shared_files += [x[len("eccodes/") :] for x in shared("eccodes/copying")] - - if os.name == "nt": - for n in os.listdir("eccodes"): - if n.endswith(".dll"): - shared_files.append(n) - -else: - ext_modules = [] - shared_files = [] - - -install_requires = ["numpy"] -if sys.version_info < (3, 7): - install_requires = ["numpy<1.20"] -elif sys.version_info < (3, 8): - install_requires = ["numpy<1.22"] -elif sys.version_info < (3, 9): - install_requires = ["numpy<1.25"] - -install_requires += ["attrs", "cffi", "findlibs"] - setuptools.setup( name="eccodes", - version=parse_version_from("gribapi/bindings.py"), - description="Python interface to the ecCodes GRIB and BUFR decoder/encoder", - long_description=read("README.rst") + read("CHANGELOG.rst"), - author="European Centre for Medium-Range Weather Forecasts (ECMWF)", - author_email="software.support@ecmwf.int", - license="Apache License Version 2.0", - url="https://github.com/ecmwf/eccodes-python", + version=get_version(), packages=setuptools.find_packages(), - include_package_data=True, - package_data={"": shared_files}, - install_requires=install_requires, + package_data={"": ["**/*.h"]}, + install_requires=parse_dependencies() + install_requires, + **ext_kwargs[sys.platform], + # NOTE what is this? Setuptools 75.6.0 doesnt recognize. Move to extras? tests_require=[ "pytest", "pytest-cov", From d148455e656ebef9c616b0b52250bc40efdd33ca Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 16 Dec 2024 13:14:37 +0100 Subject: [PATCH 2/8] Improve install_requires --- setup.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 2dce59b8..5d588287 100644 --- a/setup.py +++ b/setup.py @@ -16,16 +16,16 @@ from setup_utils import parse_dependencies, ext_kwargs -if sys.version_info < (3, 7): - install_requires = ["numpy<1.20"] -elif sys.version_info < (3, 8): - install_requires = ["numpy<1.22"] -elif sys.version_info < (3, 9): - install_requires = ["numpy<1.25"] -else: - install_requires = ["numpy"] +install_requires = [ + "numpy<1.20 ; python_version < '3.7'", + "numpy<1.22 ; python_version >= '3.7' and python_version < '3.8'", + "numpy<1.25 ; python_version >= '3.8' and python_version < '3.9'", + "numpy ; python_version >= '3.9'", + "attrs", + "cffi", + "findlibs", # TODO add lb here once released +] -install_requires += ["attrs", "cffi", "findlibs"] ext_modules = [ setuptools.Extension( "eccodes._eccodes", From d700baefe0607aae8edfb6037a0a28c979b9bc01 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 19 May 2025 12:52:23 +0200 Subject: [PATCH 3/8] Reflect new wheelmaker changes --- .github/workflows/build-wheel-linux.yml | 184 ++++++---------------- .github/workflows/build-wheel-macos.yml | 200 +++++++----------------- buildconfig | 14 -- gribapi/bindings.py | 2 +- setup.cfg | 4 +- setup.py | 59 ++++--- 6 files changed, 136 insertions(+), 327 deletions(-) delete mode 100644 buildconfig diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index 3efd4199..8a66b118 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -11,153 +11,65 @@ name: Build Python Wheel for Linux on: # Trigger the workflow manually - workflow_dispatch: ~ - - # Allow to be called from another workflow - workflow_call: ~ - - # TODO automation trigger - # repository_dispatch: - # types: [eccodes-updated] + workflow_dispatch: + inputs: + use_test_pypi: + description: Use test pypi instead of the regular one + required: false + type: boolean + default: false + + # Allow to be called from another workflow -- eg `cd.yml` + workflow_call: + inputs: + use_test_pypi: + description: Use test pypi instead of the regular one + required: false + type: boolean + default: false jobs: - build-python-wheel: + build: name: Build manylinux_2_28 - runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - container: - image: dockcross/manylinux_2_28-x64:20250109-7bf589c - #options: --pull always - - name: Build manylinux_2_28-x64 - - steps: - - uses: actions/checkout@v4 - - - run: ./scripts/build-linux.sh - - ################################################################ - - run: ./scripts/wheel-linux.sh 3.8 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.8 - with: - name: wheel-manylinux2014-3.8 - path: wheelhouse/*.whl - - # ################################################################ - - run: ./scripts/wheel-linux.sh 3.9 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.9 - with: - name: wheel-manylinux2014-3.9 - path: wheelhouse/*.whl - - # ################################################################ - - run: ./scripts/wheel-linux.sh 3.10 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.10 - with: - name: wheel-manylinux2014-3.10 - path: wheelhouse/*.whl - - # ################################################################ - - run: ./scripts/wheel-linux.sh 3.11 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.11 - with: - name: wheel-manylinux2014-3.11 - path: wheelhouse/*.whl - - # ################################################################ - - run: ./scripts/wheel-linux.sh 3.12 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.12 - with: - name: wheel-manylinux2014-3.12 - path: wheelhouse/*.whl - - # ################################################################ - - run: ./scripts/wheel-linux.sh 3.13 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.13 - with: - name: wheel-manylinux2014-3.13 - path: wheelhouse/*.whl - - test: - - needs: build - strategy: - fail-fast: false - matrix: # We don't test 3.6, as it is not supported anymore by github actions - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] - - runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - - name: Test with ${{ matrix.python-version }} - - steps: - - - uses: actions/checkout@v4 - - - uses: actions/download-artifact@v4 - with: - name: wheel-manylinux2014-${{ matrix.python-version }} - - - run: ./scripts/test-linux.sh ${{ matrix.python-version }} - - - deploy: - - if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }} - - strategy: - fail-fast: false + fail-fast: true matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] - - needs: [test, build] - - name: Deploy wheel ${{ matrix.python-version }} - - runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - - - steps: - - - run: mkdir artifact-${{ matrix.python-version }} - - - uses: actions/checkout@v4 - - - uses: actions/download-artifact@v4 - with: - name: wheel-manylinux2014-${{ matrix.python-version }} - path: artifact-${{ matrix.python-version }} - - build-wrapper-wheel: - name: Build manylinux_2_28 + python_version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] container: - image: eccr.ecmwf.int/wheelmaker/2_28:latest + image: eccr.ecmwf.int/wheelmaker/2_28:1.latest credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} steps: - # TODO convert this to be matrix-friendly. Note it's a bit tricky since - # we'd ideally not reexecute the compile step multiple times, but it - # (non-essentially) depends on a matrix-based step - # NOTE we dont use action checkout because it doesnt cleanup after itself correctly - - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj - - run: cd /proj && /buildscripts/prepare_deps.sh ./buildconfig 3.11 - run: | - cd /proj - export PYTHONPATH=/buildscripts/; export LIBDIR=/tmp/prereqs/eccodeslib/lib64; export INCDIR=/tmp/prereqs/eccodeslib/include - uv run --python python3.11 python -m build --installer uv --wheel . - - run: mkdir -p /build/wheel && mv /proj/dist/*whl /build/wheel - - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /build/wheel/*whl - - run: cd /proj && /buildscripts/upload-pypi.sh /build/wheel/*whl + set -euo pipefail + git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /src/eccodes-python + cd /src/eccodes-python + if [ "$GITHUB_REF_NAME" != "main" -a "$GITHUB_REF_NAME" != "master" -a "$GITHUB_REF_TYPE" != "tag" ] ; then + export UV_CACHE_DIR="/tmp/reallynocache" + rm -rf $UV_CACHE_DIR && mkdir $UV_CACHE_DIR + EXTRA_PIP="--refresh --no-cache --prerelease=allow" + else + EXTRA_PIP="" + fi + + VENV_ROOT=/venv/build"${{ matrix.python_version }}" + source $VENV_ROOT/bin/activate + uv pip install $EXTRA_PIP eccodeslib + EL_ROOT=$VENV_ROOT/lib/python"${{ matrix.python_version }}"/site-packages/eccodeslib + export LIBDIR=$EL_ROOT/lib64 + export INCDIR=$EL_ROOT/include + python -m build --no-isolation . + + uv pip install $EXTRA_PIP ./dist/*whl pytest + cd tests + ECCODES_PYTHON_TRACE_LIB_SEARCH=1 pytest -v -s + cd .. + + if [ "${{ inputs.use_test_pypi }}" = "true" ] ; then UPLOAD_TO=test ; else UPLOAD_TO=prod ; fi + PYTHONPATH=/buildscripts /buildscripts/upload-pypi.sh $UPLOAD_TO ./dist env: TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - # NOTE temporary thing until all the mess gets cleared - - run: rm -rf ./* ./.git ./.github + TWINE_PASSWORD_PROD: ${{ secrets.PYPI_API_TOKEN }} + TWINE_PASSWORD_TEST: ${{ secrets.PYPI_TEST_API_TOKEN }} diff --git a/.github/workflows/build-wheel-macos.yml b/.github/workflows/build-wheel-macos.yml index cd6c1a3e..6c696c8a 100644 --- a/.github/workflows/build-wheel-macos.yml +++ b/.github/workflows/build-wheel-macos.yml @@ -6,163 +6,77 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. -name: Build MacOS ARM + +name: Build Python Wheel for MacOS on: # Trigger the workflow manually - workflow_dispatch: ~ - - # allow to be called from another workflow - workflow_call: ~ - - # repository_dispatch: - # types: [eccodes-updated] - - push: - tags-ignore: - - '**' - paths: - - 'scripts/common.sh' - - 'scripts/select-python-macos.sh' - - 'scripts/build-macos.sh' - - 'scripts/wheel-macos.sh' - - 'scripts/test-macos.sh' - - 'scripts/copy-licences.py' - - '.github/workflows/build-wheel-macos.yml' - -# We don't use "actions/setup-python@v4" as it installs a universal python -# which creates universal wheels. We want to create wheels for the specific -# architecture we are running on. + workflow_dispatch: + inputs: + use_test_pypi: + description: Use test pypi instead of the regular one + required: false + type: boolean + default: false + + # Allow to be called from another workflow -- eg `cd.yml` + workflow_call: + inputs: + use_test_pypi: + description: Use test pypi instead of the regular one + required: false + type: boolean + default: false jobs: - build: - - # if: false # for temporarily disabling for debugging - - strategy: - matrix: - arch_type: [ARM64, X64] - runs-on: [self-hosted, macOS, "${{ matrix.arch_type }}"] - - name: Build - - steps: - - - run: sudo mkdir -p /Users/runner - - run: sudo chown administrator:staff /Users/runner - - - uses: actions/checkout@v2 - - - run: ./scripts/build-macos.sh "3.10" - - - run: ./scripts/wheel-macos.sh "3.9" - - run: ls -l wheelhouse - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.9 ${{ matrix.arch_type }} - with: - name: wheel-macos-${{ matrix.arch_type }}-3.9 - path: wheelhouse/*.whl - - run: rm -fr wheelhouse - - - run: ./scripts/wheel-macos.sh "3.10" - - run: ls -l wheelhouse - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.10 ${{ matrix.arch_type }} - with: - name: wheel-macos-${{ matrix.arch_type }}-3.10 - path: wheelhouse/*.whl - - run: rm -fr wheelhouse - - - run: ./scripts/wheel-macos.sh "3.11" - - run: ls -l wheelhouse - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.11 ${{ matrix.arch_type }} - with: - name: wheel-macos-${{ matrix.arch_type }}-3.11 - path: wheelhouse/*.whl - - run: rm -fr wheelhouse - - - run: ./scripts/wheel-macos.sh "3.12" - - run: ls -l wheelhouse - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.12 ${{ matrix.arch_type }} - with: - name: wheel-macos-${{ matrix.arch_type }}-3.12 - path: wheelhouse/*.whl - - run: rm -fr wheelhouse - - - run: ./scripts/wheel-macos.sh "3.13" - - run: ls -l wheelhouse - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.13 ${{ matrix.arch_type }} - with: - name: wheel-macos-${{ matrix.arch_type }}-3.13 - path: wheelhouse/*.whl - - run: rm -fr wheelhouse - - test: - needs: build - - strategy: - fail-fast: true - max-parallel: 1 - matrix: - arch_type: [ARM64, X64] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - - runs-on: [self-hosted, macOS, "${{ matrix.arch_type }}"] - - name: Test with Python ${{ matrix.python-version }} ${{ matrix.arch_type }} - - steps: - - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v4 - with: - name: wheel-macos-${{ matrix.arch_type }}-${{ matrix.python-version }} - - - run: ./scripts/test-macos.sh ${{ matrix.python-version }} - - - deploy: - - if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }} - - needs: [test, build] - - name: Deploy wheel ${{ matrix.python-version }} ${{ matrix.arch_type }} - + name: Build macos wheel strategy: fail-fast: true - max-parallel: 1 matrix: arch_type: [ARM64, X64] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - + python_version: ["3.9", "3.10", "3.11", "3.12", "3.13"] runs-on: [self-hosted, macOS, "${{ matrix.arch_type }}"] - steps: - - - run: mkdir artifact-${{ matrix.arch_type }}-${{ matrix.python-version }} - + - run: if [ -z "$(which uv)" ] ; then curl -LsSf https://astral.sh/uv/install.sh | sh ; fi - uses: actions/checkout@v4 - - - uses: actions/download-artifact@v4 with: - name: wheel-macos-${{ matrix.arch_type }}-${{ matrix.python-version }} - path: artifact-${{ matrix.arch_type }}-${{ matrix.python-version }} - + repository: ecmwf/ci-utils + ref: 1.latest + path: ci-utils + token: ${{ secrets.GH_REPO_READ_TOKEN }} - run: | - source ./scripts/select-python-macos.sh ${{ matrix.python-version }} - VENV_DIR=./dist_venv_${{ matrix.python-version }} - rm -rf ${VENV_DIR} - python3 -m venv ${VENV_DIR} - source ${VENV_DIR}/bin/activate - pip3 install twine - ls -l artifact-${{ matrix.arch_type }}-${{ matrix.python-version }}/*.whl - twine upload artifact-${{ matrix.arch_type }}-${{ matrix.python-version }}/*.whl + set -euo pipefail + git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY ./eccodes-python + cd ./eccodes-python + if [ "$GITHUB_REF_NAME" != "main" -a "$GITHUB_REF_NAME" != "master" -a "$GITHUB_REF_TYPE" != "tag" ] ; then + export UV_CACHE_DIR="/tmp/reallynocache" + rm -rf $UV_CACHE_DIR && mkdir $UV_CACHE_DIR + EXTRA_PIP="--refresh --no-cache --prerelease=allow" + else + EXTRA_PIP="" + fi + + VENV_ROOT=/tmp/buildvenv + uv python install python"${{ matrix.python_version }}" + # NOTE twine version forced due to metadata issue, cf wheelmaker Dockerfile + rm -rf $VENV_ROOT && uv venv --python python"${{ matrix.python_version }}" $VENV_ROOT && source $VENV_ROOT/bin/activate && uv pip install build twine==6.0.1 delocate setuptools requests + + uv pip install $EXTRA_PIP eccodeslib + EL_ROOT=$VENV_ROOT/lib/python"${{ matrix.python_version }}"/site-packages/eccodeslib + export LIBDIR=$EL_ROOT/lib + export INCDIR=$EL_ROOT/include + python -m build --no-isolation . + + uv pip install $EXTRA_PIP ./dist/*whl pytest + cd tests + ECCODES_PYTHON_TRACE_LIB_SEARCH=1 pytest -v -s + cd .. + + if [ "${{ inputs.use_test_pypi }}" = "true" ] ; then UPLOAD_TO=test ; else UPLOAD_TO=prod ; fi + BUILDSCRIPTS=$GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts + PYTHONPATH=$BUILDSCRIPTS $BUILDSCRIPTS/upload-pypi.sh $UPLOAD_TO ./dist env: TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + TWINE_PASSWORD_PROD: ${{ secrets.PYPI_API_TOKEN }} + TWINE_PASSWORD_TEST: ${{ secrets.PYPI_TEST_API_TOKEN }} diff --git a/buildconfig b/buildconfig deleted file mode 100644 index 7ad99f6c..00000000 --- a/buildconfig +++ /dev/null @@ -1,14 +0,0 @@ -# (C) Copyright 2024- ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -# NOTE since eccodes-python uses only parts of the Wheelmaker tooling, this file -# is only correspondingly partial. Namely, `compile` and `wheel-linux` options -# are not present, and only `prepare_deps` and `test-wheel` invocation is assumed - -DEPENDENCIES='["eccodeslib"]' -NAME="eccodes" diff --git a/gribapi/bindings.py b/gribapi/bindings.py index 3354cc20..12a10372 100644 --- a/gribapi/bindings.py +++ b/gribapi/bindings.py @@ -39,7 +39,7 @@ def _find_eccodes_custom() -> str|None: - # TODO deprecate this method in favour of findlibs only + # TODO delete once the wheels are eccodeslib-based for all platforms name = "eccodes" env_var = "ECCODES_PYTHON_USE_FINDLIBS" if int(os.environ.get(env_var, "0")): diff --git a/setup.cfg b/setup.cfg index c8cf29bd..84b9d014 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,15 +1,13 @@ [metadata] description = "eccodes" long_description = file: README.rst -long_description_content_type = text/rst +long_description_content_type = text/x-rst author = "European Centre for Medium-Range Weather Forecasts (ECMWF)" author_email = "software.support@ecmwf.int" -url = "https://github.com/ecmwf/eccodes-python" keywords = ecCodes, GRIB, BUFR classifiers = Development Status :: 4 - Beta Intended Audience :: Developers - License :: OSI Approved :: Apache Software License Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 diff --git a/setup.py b/setup.py index 5d588287..36db9fb0 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,20 @@ -# (C) Copyright 2024- ECMWF. +#!/usr/bin/env python +# +# (C) Copyright 2017- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -# TODO ideally merge this with wheelmaker's setup_utils somehow +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. import io +import importlib.metadata import os import re import sys import setuptools -from setup_utils import parse_dependencies, ext_kwargs +from wheel.bdist_wheel import bdist_wheel install_requires = [ @@ -23,7 +24,7 @@ "numpy ; python_version >= '3.9'", "attrs", "cffi", - "findlibs", # TODO add lb here once released + "findlibs>=0.1.1", ] ext_modules = [ @@ -48,35 +49,33 @@ def get_version() -> str: raise ValueError("couldn't parse version") return version_match.group(1) +def get_eccodeslib_dep() -> list[str]: + eccodes_version = importlib.metadata.version("eccodeslib") + mj, mn, pt = eccodes_version.split('.', 2) + return [ + f"eccodeslib >= {eccodes_version}, < {int(mj)+1}", + ] + +class bdist_wheel_ext(bdist_wheel): + # cf wheelmaker setup.py for explanation + def get_tag(self): + python, abi, plat = bdist_wheel.get_tag(self) + return python, abi, "manylinux_2_28_x86_64" + +ext_kwargs = { + 'darwin': {}, + 'linux': {"cmdclass": {"bdist_wheel": bdist_wheel_ext}}, +} + setuptools.setup( name="eccodes", version=get_version(), packages=setuptools.find_packages(), package_data={"": ["**/*.h"]}, - install_requires=parse_dependencies() + install_requires, - **ext_kwargs[sys.platform], - # NOTE what is this? Setuptools 75.6.0 doesnt recognize. Move to extras? - tests_require=[ - "pytest", - "pytest-cov", - "pytest-flakes", - ], - test_suite="tests", + install_requires=get_eccodeslib_dep() + install_requires, zip_safe=True, keywords="ecCodes GRIB BUFR", - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Operating System :: OS Independent", - ], ext_modules=ext_modules, + **ext_kwargs[sys.platform], + ) From f2df0c05ade51316fe68fe91ccac45b26edc5a94 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 26 May 2025 10:36:10 +0200 Subject: [PATCH 4/8] Preserve windows legacy --- gribapi/bindings.py | 8 +-- scripts/wheel-windows.sh | 2 +- setup_windows.py | 107 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 setup_windows.py diff --git a/gribapi/bindings.py b/gribapi/bindings.py index 12a10372..4742bfa7 100644 --- a/gribapi/bindings.py +++ b/gribapi/bindings.py @@ -38,8 +38,8 @@ LOG.addHandler(logging.StreamHandler()) -def _find_eccodes_custom() -> str|None: - # TODO delete once the wheels are eccodeslib-based for all platforms +def _find_eccodes_windows() -> str|None: + # TODO delete once windows ceases to be supported name = "eccodes" env_var = "ECCODES_PYTHON_USE_FINDLIBS" if int(os.environ.get(env_var, "0")): @@ -85,7 +85,9 @@ def _find_eccodes_custom() -> str|None: return None -library_path = _find_eccodes_custom() +library_path = None +if sys.platform == "win32": + library_path = _find_eccodes_windows() if library_path is None: import findlibs library_path = findlibs.find("eccodes") diff --git a/scripts/wheel-windows.sh b/scripts/wheel-windows.sh index 9fb6fb66..04e0de5b 100644 --- a/scripts/wheel-windows.sh +++ b/scripts/wheel-windows.sh @@ -13,7 +13,7 @@ set -eaux pip install wheel setuptools rm -fr dist wheelhouse ecmwflibs.egg-info build -python setup.py --binary-wheel bdist_wheel +python setup_windows.py --binary-wheel bdist_wheel mv dist wheelhouse ls -l wheelhouse diff --git a/setup_windows.py b/setup_windows.py new file mode 100644 index 00000000..cb3260fc --- /dev/null +++ b/setup_windows.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# +# (C) Copyright 2017- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. +# + +import io +import os +import re +import sys + +import setuptools + + +def read(path): + file_path = os.path.join(os.path.dirname(__file__), *path.split("/")) + return io.open(file_path, encoding="utf-8").read() + + +# single-sourcing the package version using method 1 of: +# https://packaging.python.org/guides/single-sourcing-package-version/ +def parse_version_from(path): + version_pattern = ( + r"^__version__ = [\"\'](.*)[\"\']" # More permissive regex pattern + ) + version_file = read(path) + version_match = re.search(version_pattern, version_file, re.M) + if version_match is None or len(version_match.groups()) > 1: + raise ValueError("couldn't parse version") + return version_match.group(1) + + +# for the binary wheel +libdir = os.path.realpath("install/lib") +incdir = os.path.realpath("install/include") +libs = ["eccodes"] + +if "--binary-wheel" in sys.argv: + sys.argv.remove("--binary-wheel") + + # https://setuptools.pypa.io/en/latest/userguide/ext_modules.html + ext_modules = [ + setuptools.Extension( + "eccodes._eccodes", + sources=["eccodes/_eccodes.cc"], + language="c++", + libraries=libs, + library_dirs=[libdir], + include_dirs=[incdir], + extra_link_args=["-Wl,-rpath," + libdir], + ) + ] + + def shared(directory): + result = [] + for path, dirs, files in os.walk(directory): + for f in files: + result.append(os.path.join(path, f)) + return result + + # Paths must be relative to package directory... + shared_files = ["versions.txt"] + shared_files += [x[len("eccodes/") :] for x in shared("eccodes/copying")] + + if os.name == "nt": + for n in os.listdir("eccodes"): + if n.endswith(".dll"): + shared_files.append(n) + +else: + ext_modules = [] + shared_files = [] + + +install_requires = ["numpy"] +if sys.version_info < (3, 7): + install_requires = ["numpy<1.20"] +elif sys.version_info < (3, 8): + install_requires = ["numpy<1.22"] +elif sys.version_info < (3, 9): + install_requires = ["numpy<1.25"] + +install_requires += ["attrs", "cffi", "findlibs"] + +setuptools.setup( + name="eccodes", + version=parse_version_from("gribapi/bindings.py"), + packages=setuptools.find_packages(), + include_package_data=True, + package_data={"": shared_files}, + install_requires=install_requires, + tests_require=[ + "pytest", + "pytest-cov", + "pytest-flakes", + ], + test_suite="tests", + zip_safe=True, + keywords="ecCodes GRIB BUFR", + ext_modules=ext_modules, +) From e417f3c7470fdd05419b7995e7a5c6fec7076804 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 2 Jun 2025 14:55:16 +0200 Subject: [PATCH 5/8] Fix isort --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 36db9fb0..86bc647a 100644 --- a/setup.py +++ b/setup.py @@ -8,15 +8,15 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. -import io import importlib.metadata +import io import os import re import sys + import setuptools from wheel.bdist_wheel import bdist_wheel - install_requires = [ "numpy<1.20 ; python_version < '3.7'", "numpy<1.22 ; python_version >= '3.7' and python_version < '3.8'", From 0d1501c952edfde710becc98cc3b77ca4b8cb547 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 2 Jun 2025 14:59:14 +0200 Subject: [PATCH 6/8] Fix black --- gribapi/bindings.py | 3 ++- setup.py | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/gribapi/bindings.py b/gribapi/bindings.py index 4742bfa7..d72f84fc 100644 --- a/gribapi/bindings.py +++ b/gribapi/bindings.py @@ -38,7 +38,7 @@ LOG.addHandler(logging.StreamHandler()) -def _find_eccodes_windows() -> str|None: +def _find_eccodes_windows() -> str | None: # TODO delete once windows ceases to be supported name = "eccodes" env_var = "ECCODES_PYTHON_USE_FINDLIBS" @@ -90,6 +90,7 @@ def _find_eccodes_windows() -> str|None: library_path = _find_eccodes_windows() if library_path is None: import findlibs + library_path = findlibs.find("eccodes") LOG.debug(f"eccodes lib search: findlibs returned {library_path}") if library_path is None: diff --git a/setup.py b/setup.py index 86bc647a..0573708f 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ ) ] + def get_version() -> str: version_pattern = ( r"^__version__ = [\"\'](.*)[\"\']" # More permissive regex pattern @@ -49,22 +50,25 @@ def get_version() -> str: raise ValueError("couldn't parse version") return version_match.group(1) + def get_eccodeslib_dep() -> list[str]: eccodes_version = importlib.metadata.version("eccodeslib") - mj, mn, pt = eccodes_version.split('.', 2) + mj, mn, pt = eccodes_version.split(".", 2) return [ f"eccodeslib >= {eccodes_version}, < {int(mj)+1}", ] + class bdist_wheel_ext(bdist_wheel): # cf wheelmaker setup.py for explanation def get_tag(self): python, abi, plat = bdist_wheel.get_tag(self) return python, abi, "manylinux_2_28_x86_64" + ext_kwargs = { - 'darwin': {}, - 'linux': {"cmdclass": {"bdist_wheel": bdist_wheel_ext}}, + "darwin": {}, + "linux": {"cmdclass": {"bdist_wheel": bdist_wheel_ext}}, } setuptools.setup( @@ -77,5 +81,4 @@ def get_tag(self): keywords="ecCodes GRIB BUFR", ext_modules=ext_modules, **ext_kwargs[sys.platform], - ) From bb9efbf60735b462e6deac81f6b72a659c29dc77 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Tue, 3 Jun 2025 08:44:10 +0200 Subject: [PATCH 7/8] Support ci-based pipeline --- .pre-commit-config.yaml | 13 +++++++++++++ setup.py | 38 +++++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..8c62281f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.8.0 + hooks: + - id: black +- repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + args: + - --profile black +ci: + autoupdate_schedule: monthly diff --git a/setup.py b/setup.py index 0573708f..c79a7def 100644 --- a/setup.py +++ b/setup.py @@ -25,18 +25,23 @@ "attrs", "cffi", "findlibs>=0.1.1", + 'eccodeslib ; platform_system!="Windows"', ] -ext_modules = [ - setuptools.Extension( - "eccodes._eccodes", - sources=["eccodes/_eccodes.cc"], - language="c++", - libraries=["eccodes"], - library_dirs=[os.environ["LIBDIR"]], - include_dirs=[os.environ["INCDIR"]], - ) -] +if os.environ.get("LIBDIR", None): + ext_modules = [ + setuptools.Extension( + "eccodes._eccodes", + sources=["eccodes/_eccodes.cc"], + language="c++", + libraries=["eccodes"], + library_dirs=[os.environ["LIBDIR"]], + include_dirs=[os.environ["INCDIR"]], + ) + ] +else: + # NOTE this hack is due to downstream CI not yet supporting building + ext_modules = [] def get_version() -> str: @@ -52,11 +57,14 @@ def get_version() -> str: def get_eccodeslib_dep() -> list[str]: - eccodes_version = importlib.metadata.version("eccodeslib") - mj, mn, pt = eccodes_version.split(".", 2) - return [ - f"eccodeslib >= {eccodes_version}, < {int(mj)+1}", - ] + try: + eccodes_version = importlib.metadata.version("eccodeslib") + mj, mn, pt = eccodes_version.split(".", 2) + return [ + f"eccodeslib >= {eccodes_version}, < {int(mj)+1}", + ] + except importlib.metadata.PackageNotFoundError: + return [] class bdist_wheel_ext(bdist_wheel): From 43c27372504fa2244039a924405a167899dbafd9 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Tue, 3 Jun 2025 09:58:38 +0200 Subject: [PATCH 8/8] Add debug test logging to troubleshoot segfault --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytest.ini b/pytest.ini index fd633da0..ea6bca50 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,3 +2,6 @@ ; addopts=-s --cov climetlab --verbose --cov-report xml --cov-report html ; addopts=--no-cov addopts=-s --verbose +log_cli = true +; for some reason, "DEBUG" was not accepted to log_cli_level so we use 10 +log_cli_level = 10