Skip to content

Commit 252f135

Browse files
jim22kalugowski
jim22k
and
alugowski
authored
Build non-Arm wheels using CI (#73)
* Setup cibuildwheel with Linux, macOS, Windows wheels * Ignore GraphBLAS-{version}/ checkout folder to avoid modifying the version * Disable less common dtypes for smaller package * Remove mac arm options --------- Co-authored-by: Adam Lugowski <[email protected]>
1 parent 49f1b70 commit 252f135

File tree

8 files changed

+264
-33
lines changed

8 files changed

+264
-33
lines changed

‎.github/workflows/wheels.yml

+156-27
Original file line numberDiff line numberDiff line change
@@ -4,36 +4,165 @@ on:
44
release:
55
types: [created]
66

7+
# Enable Run Workflow button in GitHub UI
8+
workflow_dispatch:
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.ref }}
12+
cancel-in-progress: true
13+
714
jobs:
8-
wheels:
15+
build_sdist:
16+
name: Build SDist
17+
runs-on: ubuntu-latest
18+
steps:
19+
- uses: actions/checkout@v3
20+
with:
21+
fetch-depth: 0
22+
23+
- name: Build SDist
24+
run: pipx run build --sdist
25+
26+
- name: Check metadata
27+
run: pipx run twine check dist/*
28+
29+
- uses: actions/upload-artifact@v3
30+
with:
31+
path: dist/*.tar.gz
32+
33+
34+
build_wheels:
35+
name: Wheels on ${{ matrix.platform_id }} - ${{ matrix.os }}
936
runs-on: ${{ matrix.os }}
10-
defaults:
11-
run:
12-
shell: bash -l {0}
1337
strategy:
1438
fail-fast: false
1539
matrix:
16-
os: ["ubuntu-latest"]
40+
# Loosely based on scikit-learn's config:
41+
# https://github.com/scikit-learn/scikit-learn/blob/main/.github/workflows/wheels.yml
42+
include:
43+
- os: windows-latest
44+
python-version: "3.8"
45+
platform_id: win_amd64
46+
47+
# Linux 64 bit manylinux2014
48+
- os: ubuntu-latest
49+
python-version: "3.8"
50+
platform_id: manylinux_x86_64
51+
manylinux_image: manylinux2014
52+
53+
# Use x86 macOS runner to build both x86 and ARM. GitHub does not offer M1/M2 yet (only self-hosted).
54+
- os: macos-latest
55+
python-version: "3.8"
56+
platform_id: macosx_x86_64
57+
1758
steps:
18-
- name: Checkout
19-
uses: actions/checkout@v3
20-
- name: Set up Python
21-
uses: actions/setup-python@v4
22-
with:
23-
python-version: '3.8'
24-
- name: Upgrade pip
25-
run: |
26-
python -m pip install --upgrade pip
27-
- name: Build manylinux Python wheels
28-
uses: RalfG/[email protected]_x86_64
29-
with:
30-
python-versions: 'cp38-cp38 cp39-cp39'
31-
build-requirements: 'cffi numpy>=1.19,<1.20 cython'
32-
pre-build-command: ${{ format('sh suitesparse.sh {0}', github.ref) }}
33-
- name: Publish wheels to PyPI
34-
env:
35-
TWINE_USERNAME: __token__
36-
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
37-
run: |
38-
pip install twine
39-
twine upload dist/*-manylinux*.whl
59+
- uses: actions/checkout@v3
60+
with:
61+
fetch-depth: 0
62+
63+
- uses: actions/setup-python@v4
64+
with:
65+
python-version: ${{ matrix.python-version }}
66+
67+
# - name: Install tools (macOS)
68+
# if: contains(matrix.os, 'macos')
69+
# # Install coreutils which includes `nproc` used by `make -j` in suitesparse.sh
70+
# #
71+
# # GitHub actions comes with libomp already installed, but for its native arch only. Must build universal one
72+
# # manually so that both x86 and arm builds can be built.
73+
# run: |
74+
# brew install coreutils
75+
# brew install libomp
76+
# sh add_arm_to_libomp_dylib.sh
77+
78+
- name: Build Wheels
79+
env:
80+
# very verbose
81+
CIBW_BUILD_VERBOSITY: 3
82+
83+
# Build SuiteSparse
84+
CIBW_BEFORE_ALL: bash suitesparse.sh ${{ github.ref }}
85+
86+
# CMAKE_GNUtoMS=ON asks suitesparse.sh to build libraries in MSVC style on Windows.
87+
CIBW_ENVIRONMENT_WINDOWS: CMAKE_GNUtoMS=ON GRAPHBLAS_PREFIX="C:/GraphBLAS"
88+
89+
# macOS libomp requires special configs. BREW_LIBOMP=1 asks suitesparse.sh to include them.
90+
CIBW_ENVIRONMENT_MACOS: BREW_LIBOMP="1"
91+
92+
# Uncomment to only build CPython wheels
93+
# CIBW_BUILD: "cp*"
94+
95+
# macOS: build x86_64 and arm64
96+
#CIBW_ARCHS_MACOS: "x86_64 arm64"
97+
98+
# No 32-bit builds
99+
CIBW_SKIP: "*-win32 *_i686 *musl*"
100+
101+
# Use delvewheel on Windows.
102+
# This copies graphblas.dll into the wheel. "repair" in cibuildwheel parlance includes copying any shared
103+
# libraries from the build host into the wheel to make the wheel self-contained.
104+
# Cibuildwheel includes tools for this for Linux and macOS, and they recommend delvewheel for Windows.
105+
# Note: Currently using a workaround: --no-mangle instead of stripping graphblas.dll
106+
# see https://github.com/adang1345/delvewheel/issues/33
107+
CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel"
108+
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path \"C:\\GraphBLAS\\bin\" --no-mangle \"libgomp-1.dll;libgcc_s_seh-1.dll\" -w {dest_dir} {wheel}"
109+
110+
# make cibuildwheel install test dependencies from pyproject.toml
111+
CIBW_TEST_EXTRAS: "test"
112+
113+
# run tests
114+
CIBW_TEST_COMMAND: "pytest {project}/suitesparse_graphblas/tests"
115+
116+
# GitHub Actions macOS Intel runner cannot run ARM tests.
117+
CIBW_TEST_SKIP: "*-macosx_arm64"
118+
119+
run: |
120+
python -m pip install cibuildwheel
121+
python -m cibuildwheel --output-dir wheelhouse .
122+
shell: bash
123+
124+
- uses: actions/upload-artifact@v3
125+
id: uploadAttempt1
126+
continue-on-error: true
127+
with:
128+
path: wheelhouse/*.whl
129+
if-no-files-found: error
130+
131+
# Retry upload if first attempt failed. This happens somewhat randomly and for irregular reasons.
132+
# Logic is a duplicate of previous step.
133+
- uses: actions/upload-artifact@v3
134+
id: uploadAttempt2
135+
if: steps.uploadAttempt1.outcome == 'failure'
136+
continue-on-error: false
137+
with:
138+
path: wheelhouse/*.whl
139+
if-no-files-found: error
140+
141+
upload_all:
142+
name: Upload to PyPI
143+
needs: [build_wheels, build_sdist]
144+
runs-on: ubuntu-latest
145+
if: github.repository == 'GraphBLAS/python-suitesparse-graphblas'
146+
# if: github.event_name == 'release' && github.event.action == 'published'
147+
148+
steps:
149+
- uses: actions/setup-python@v4
150+
with:
151+
python-version: "3.x"
152+
153+
- uses: actions/download-artifact@v3
154+
with:
155+
name: artifact
156+
path: dist
157+
158+
- uses: pypa/gh-action-pypi-publish@release/v1
159+
with:
160+
# PyPI does not allow replacing a file. Without this flag the entire action fails if even a single duplicate exists.
161+
skip_existing: true
162+
verbose: true
163+
# Real PyPI:
164+
password: ${{ secrets.PYPI_TOKEN }}
165+
166+
# Test PyPI:
167+
# password: ${{ secrets.TEST_PYPI_API_TOKEN }}
168+
# repository_url: https://test.pypi.org/legacy/

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ share/python-wheels/
3030
MANIFEST
3131
wheelhouse
3232

33+
# Wheel building stuff
34+
GraphBLAS-*/
35+
3336
# PyInstaller
3437
# Usually these files are written by a python script from a template
3538
# before PyInstaller builds the exe, so as to inject date/other infos into it.

‎add_arm_to_libomp_dylib.sh

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/sh
2+
3+
#mkdir x86lib
4+
mkdir armlib
5+
6+
# download and unzip both x86 and arm libomp tarballs
7+
#brew fetch --force --bottle-tag=x86_64_monterey libomp
8+
brew fetch --force --bottle-tag=arm64_big_sur libomp
9+
10+
# untar
11+
#tar -xzf $(brew --cache --bottle-tag=x86_64_monterey libomp) --strip-components 2 -C x86lib
12+
tar -xzf $(brew --cache --bottle-tag=arm64_big_sur libomp) --strip-components 2 -C armlib
13+
14+
# merge
15+
lipo armlib/lib/libomp.dylib $(brew --prefix libomp)/lib/libomp.dylib -output libomp.dylib -create
16+
cp -f libomp.dylib $(brew --prefix libomp)/lib
17+
rm libomp.dylib
18+
rm -rf armlib

‎build_graphblas_cffi.py

+6
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@
1616
include_dirs.append(os.path.join(sys.prefix, "Library", "include"))
1717
library_dirs.append(os.path.join(sys.prefix, "Library", "lib"))
1818

19+
# wheels.yml configures suitesparse.sh to install GraphBLAS here.
20+
prefix = "C:\\GraphBLAS"
21+
include_dirs.append(os.path.join(prefix, "include"))
22+
library_dirs.append(os.path.join(prefix, "lib"))
23+
library_dirs.append(os.path.join(prefix, "bin"))
24+
1925
ffibuilder.set_source(
2026
"suitesparse_graphblas._graphblas",
2127
(ss_g / "source.c").read_text(),

‎pyproject.toml

+3-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ requires = [
44
"setuptools >=64",
55
"setuptools-git-versioning",
66
"wheel",
7-
"cffi",
7+
"cffi>=1.11",
88
"cython",
99
"oldest-supported-numpy",
1010
]
@@ -27,7 +27,7 @@ maintainers = [
2727
{name = "Michel Pelletier", email = "[email protected]"},
2828
]
2929
classifiers = [
30-
"Development Status :: 4 - Beta",
30+
"Development Status :: 5 - Production/Stable",
3131
"License :: OSI Approved :: Apache Software License",
3232
"Operating System :: MacOS :: MacOS X",
3333
"Operating System :: POSIX :: Linux",
@@ -45,7 +45,7 @@ classifiers = [
4545
]
4646
dependencies = [
4747
# These are super-old; can/should we update them?
48-
"cffi>=1.0.0",
48+
"cffi>=1.11",
4949
"numpy>=1.19",
5050
]
5151
[project.urls]

‎setup.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,10 @@
5858
if use_cython:
5959
ext_modules = cythonize(ext_modules, include_path=include_dirs)
6060

61-
ext_modules.append(build_graphblas_cffi.get_extension(extra_compile_args=extra_compile_args))
61+
if build_graphblas_cffi.is_win:
62+
ext_modules.append(build_graphblas_cffi.get_extension(extra_compile_args=extra_compile_args))
6263

6364
setup(
6465
ext_modules=ext_modules,
66+
cffi_modules=None if build_graphblas_cffi.is_win else ["build_graphblas_cffi.py:ffibuilder"],
6567
)

‎suitesparse.sh

100644100755
+75-2
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,87 @@
1+
#!/bin/bash
12

3+
# parse SuiteSparse version from first argument, a git tag that ends in the version (no leading v)
24
if [[ $1 =~ refs/tags/([0-9]*\.[0-9]*\.[0-9]*)\..*$ ]];
35
then
46
VERSION=${BASH_REMATCH[1]}
57
else
8+
echo "Specify a SuiteSparse version, such as: $0 refs/tags/7.4.3.0"
69
exit -1
710
fi
811
echo VERSION: $VERSION
912

13+
NPROC="$(nproc)"
14+
if [ -z "${NPROC}" ]; then
15+
# Default for platforms that don't have nproc. Mostly Windows.
16+
NPROC="2"
17+
fi
18+
19+
cmake_params=()
20+
if [ -n "${BREW_LIBOMP}" ]; then
21+
# macOS OpenMP flags.
22+
# FindOpenMP doesn't find brew's libomp, so set the necessary configs manually.
23+
cmake_params+=(-DOpenMP_C_FLAGS="-Xclang -fopenmp -I$(brew --prefix libomp)/include")
24+
cmake_params+=(-DOpenMP_C_LIB_NAMES="libomp")
25+
cmake_params+=(-DOpenMP_libomp_LIBRARY="omp")
26+
export LDFLAGS="-L$(brew --prefix libomp)/lib"
27+
28+
export CFLAGS="-arch x86_64"
29+
# # build both x86 and ARM
30+
# export CFLAGS="-arch x86_64 -arch arm64"
31+
fi
32+
33+
if [ -n "${CMAKE_GNUtoMS}" ]; then
34+
# Windows needs .lib libraries, not .a
35+
cmake_params+=(-DCMAKE_GNUtoMS=ON)
36+
# Windows expects 'graphblas.lib', not 'libgraphblas.lib'
37+
cmake_params+=(-DCMAKE_SHARED_LIBRARY_PREFIX=)
38+
cmake_params+=(-DCMAKE_STATIC_LIBRARY_PREFIX=)
39+
fi
40+
41+
if [ -n "${GRAPHBLAS_PREFIX}" ]; then
42+
echo "GRAPHBLAS_PREFIX=${GRAPHBLAS_PREFIX}"
43+
cmake_params+=(-DCMAKE_INSTALL_PREFIX="${GRAPHBLAS_PREFIX}")
44+
fi
45+
1046
curl -L https://github.com/DrTimothyAldenDavis/GraphBLAS/archive/refs/tags/v${VERSION}.tar.gz | tar xzf -
1147
cd GraphBLAS-${VERSION}/build
12-
cmake .. -DCMAKE_BUILD_TYPE=Release
13-
make -j$(nproc)
48+
49+
# Disable optimizing some rarely-used types for significantly faster builds and significantly smaller wheel size.
50+
# Also the build with all types enabled sometimes stalls on GitHub Actions. Probably due to exceeded resource limits.
51+
# These can still be used, they'll just have reduced performance (AFAIK similar to UDTs).
52+
# echo "#define GxB_NO_BOOL 1" >> ../Source/GB_control.h #
53+
# echo "#define GxB_NO_FP32 1" >> ../Source/GB_control.h #
54+
# echo "#define GxB_NO_FP64 1" >> ../Source/GB_control.h #
55+
echo "#define GxB_NO_FC32 1" >> ../Source/GB_control.h #
56+
echo "#define GxB_NO_FC64 1" >> ../Source/GB_control.h #
57+
# echo "#define GxB_NO_INT16 1" >> ../Source/GB_control.h
58+
# echo "#define GxB_NO_INT32 1" >> ../Source/GB_control.h
59+
# echo "#define GxB_NO_INT64 1" >> ../Source/GB_control.h #
60+
# echo "#define GxB_NO_INT8 1" >> ../Source/GB_control.h
61+
echo "#define GxB_NO_UINT16 1" >> ../Source/GB_control.h
62+
echo "#define GxB_NO_UINT32 1" >> ../Source/GB_control.h
63+
# echo "#define GxB_NO_UINT64 1" >> ../Source/GB_control.h
64+
echo "#define GxB_NO_UINT8 1" >> ../Source/GB_control.h
65+
66+
# Disable all Source/Generated2 kernels. For workflow development only.
67+
#cmake_params+=(-DCMAKE_CUDA_DEV=1)
68+
69+
cmake .. -DCMAKE_BUILD_TYPE=Release -G 'Unix Makefiles' "${cmake_params[@]}"
70+
make -j$NPROC
1471
make install
72+
73+
if [ -n "${CMAKE_GNUtoMS}" ]; then
74+
if [ -z "${GRAPHBLAS_PREFIX}" ]; then
75+
# Windows default
76+
GRAPHBLAS_PREFIX="C:/Program Files (x86)"
77+
fi
78+
79+
# Windows:
80+
# CMAKE_STATIC_LIBRARY_PREFIX is sometimes ignored, possibly when the MinGW toolchain is selected.
81+
# Drop the 'lib' prefix manually.
82+
echo "manually removing lib prefix"
83+
mv "${GRAPHBLAS_PREFIX}/lib/libgraphblas.lib" "${GRAPHBLAS_PREFIX}/lib/graphblas.lib"
84+
mv "${GRAPHBLAS_PREFIX}/lib/libgraphblas.dll.a" "${GRAPHBLAS_PREFIX}/lib/graphblas.dll.a"
85+
# cp instead of mv because the GNU tools expect libgraphblas.dll and the MS tools expect graphblas.dll.
86+
cp "${GRAPHBLAS_PREFIX}/bin/libgraphblas.dll" "${GRAPHBLAS_PREFIX}/bin/graphblas.dll"
87+
fi

‎suitesparse_graphblas/tests/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)