Skip to content

Commit dc5c7e6

Browse files
timfeltimfel
committed
update github actions to build wheels for popular projects
1 parent 9a2087f commit dc5c7e6

12 files changed

+1418
-770
lines changed

‎.github/workflows/build-act-wheels.yml

Lines changed: 577 additions & 0 deletions
Large diffs are not rendered by default.

‎.github/workflows/build-linux-aarch64-wheels.yml

Lines changed: 111 additions & 117 deletions
Large diffs are not rendered by default.

‎.github/workflows/build-linux-amd64-wheels.yml

Lines changed: 408 additions & 222 deletions
Large diffs are not rendered by default.

‎.github/workflows/build-macos-aarch64-wheels.yml

Lines changed: 123 additions & 179 deletions
Large diffs are not rendered by default.

‎.github/workflows/build-macos-amd64-wheels.yml

Lines changed: 123 additions & 179 deletions
Large diffs are not rendered by default.

‎.github/workflows/build-repository.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ jobs:
4747
if_no_artifact_found: warn
4848
allow_forks: 'false'
4949
- name: Set up Python
50-
uses: actions/setup-python@v4
50+
uses: actions/setup-python@v5
5151
with:
5252
python-version: '3.10'
5353
- name: Create repository

‎.github/workflows/build-windows-amd64-wheels.yml

Lines changed: 17 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,7 @@ name: build-windows-amd64-wheels
1212
required: false
1313
jobs:
1414
numpy-windows-amd64:
15-
runs-on:
16-
- self-hosted
17-
- windows
18-
- X64
15+
runs-on: windows-latest
1916
if: ${{ !cancelled() && (inputs.name == '' || inputs.name == 'numpy') }}
2017
steps:
2118
- uses: ilammy/msvc-dev-cmd@v1
@@ -24,7 +21,7 @@ jobs:
2421
Invoke-WebRequest https://kumisystems.dl.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9-7-bin.zip -OutFile patch.zip
2522
Expand-Archive patch.zip -DestinationPath ../patch -Force
2623
- name: Checkout
27-
uses: actions/checkout@v3
24+
uses: actions/checkout@main
2825
- name: Setup custom GraalPy
2926
if: inputs.graalpy != ''
3027
run: |-
@@ -35,10 +32,10 @@ jobs:
3532
graalpy/bin/graalpy.exe -s -m ensurepip
3633
graalpy/bin/graalpy.exe -m pip install wheel
3734
- name: Setup GraalPy
38-
uses: actions/setup-python@main
35+
uses: actions/setup-python@v5
3936
if: inputs.graalpy == ''
4037
with:
41-
python-version: graalpy23.1
38+
python-version: graalpy24.0
4239
- name: Setup local GraalPy venv
4340
if: inputs.graalpy == ''
4441
run: python -m venv graalpy
@@ -47,21 +44,14 @@ jobs:
4744
$env:PIP_FIND_LINKS=$PWD
4845
$env:PATH+=";$PWD\graalpy\bin;$PWD\graalpy\Scripts;$PWD\..\patch\bin"
4946
graalpy/bin/graalpy -m pip wheel --find-links $PWD numpy
50-
graalpy/bin/graalpy -m pip wheel --find-links $PWD numpy==1.21.6
51-
graalpy/bin/graalpy -m pip wheel --find-links $PWD numpy==1.22.4
52-
graalpy/bin/graalpy -m pip wheel --find-links $PWD numpy==1.23.1
53-
graalpy/bin/graalpy -m pip wheel --find-links $PWD numpy==1.23.4
5447
- name: Store wheels
55-
uses: umutozd/upload-artifact@5c459179e7745e2c730c50b10a6459da0b6f25db
48+
uses: actions/upload-artifact@main
5649
with:
5750
name: numpy-windows-amd64
5851
path: numpy*.whl
5952
if-no-files-found: error
6053
PyYAML-windows-amd64:
61-
runs-on:
62-
- self-hosted
63-
- windows
64-
- X64
54+
runs-on: windows-latest
6555
needs: numpy-windows-amd64
6656
if: ${{ !cancelled() && (inputs.name == '' || inputs.name == 'PyYAML') }}
6757
steps:
@@ -71,7 +61,7 @@ jobs:
7161
Invoke-WebRequest https://kumisystems.dl.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9-7-bin.zip -OutFile patch.zip
7262
Expand-Archive patch.zip -DestinationPath ../patch -Force
7363
- name: Checkout
74-
uses: actions/checkout@v3
64+
uses: actions/checkout@main
7565
- name: Setup custom GraalPy
7666
if: inputs.graalpy != ''
7767
run: |-
@@ -82,15 +72,15 @@ jobs:
8272
graalpy/bin/graalpy.exe -s -m ensurepip
8373
graalpy/bin/graalpy.exe -m pip install wheel
8474
- name: Setup GraalPy
85-
uses: actions/setup-python@main
75+
uses: actions/setup-python@v5
8676
if: inputs.graalpy == ''
8777
with:
88-
python-version: graalpy23.1
78+
python-version: graalpy24.0
8979
- name: Setup local GraalPy venv
9080
if: inputs.graalpy == ''
9181
run: python -m venv graalpy
9282
- name: Download artifacts from numpy
93-
uses: actions/download-artifact@v3
83+
uses: actions/download-artifact@main
9484
continue-on-error: true
9585
with:
9686
name: numpy-windows-amd64
@@ -100,16 +90,13 @@ jobs:
10090
$env:PATH+=";$PWD\graalpy\bin;$PWD\graalpy\Scripts;$PWD\..\patch\bin"
10191
graalpy/bin/graalpy -m pip wheel --find-links $PWD PyYAML
10292
- name: Store wheels
103-
uses: umutozd/upload-artifact@5c459179e7745e2c730c50b10a6459da0b6f25db
93+
uses: actions/upload-artifact@main
10494
with:
10595
name: PyYAML-windows-amd64
10696
path: PyYAML*.whl
10797
if-no-files-found: error
10898
ujson-windows-amd64:
109-
runs-on:
110-
- self-hosted
111-
- windows
112-
- X64
99+
runs-on: windows-latest
113100
needs: PyYAML-windows-amd64
114101
if: ${{ !cancelled() && (inputs.name == '' || inputs.name == 'ujson') }}
115102
steps:
@@ -119,7 +106,7 @@ jobs:
119106
Invoke-WebRequest https://kumisystems.dl.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9-7-bin.zip -OutFile patch.zip
120107
Expand-Archive patch.zip -DestinationPath ../patch -Force
121108
- name: Checkout
122-
uses: actions/checkout@v3
109+
uses: actions/checkout@main
123110
- name: Setup custom GraalPy
124111
if: inputs.graalpy != ''
125112
run: |-
@@ -130,15 +117,15 @@ jobs:
130117
graalpy/bin/graalpy.exe -s -m ensurepip
131118
graalpy/bin/graalpy.exe -m pip install wheel
132119
- name: Setup GraalPy
133-
uses: actions/setup-python@main
120+
uses: actions/setup-python@v5
134121
if: inputs.graalpy == ''
135122
with:
136-
python-version: graalpy23.1
123+
python-version: graalpy24.0
137124
- name: Setup local GraalPy venv
138125
if: inputs.graalpy == ''
139126
run: python -m venv graalpy
140127
- name: Download artifacts from PyYAML
141-
uses: actions/download-artifact@v3
128+
uses: actions/download-artifact@main
142129
continue-on-error: true
143130
with:
144131
name: PyYAML-windows-amd64
@@ -148,7 +135,7 @@ jobs:
148135
$env:PATH+=";$PWD\graalpy\bin;$PWD\graalpy\Scripts;$PWD\..\patch\bin"
149136
graalpy/bin/graalpy -m pip wheel --find-links $PWD ujson
150137
- name: Store wheels
151-
uses: umutozd/upload-artifact@5c459179e7745e2c730c50b10a6459da0b6f25db
138+
uses: actions/upload-artifact@main
152139
with:
153140
name: ujson-windows-amd64
154141
path: ujson*.whl

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,4 @@ graalpython/graalpy-maven-plugin/target
9090
graalpython/graalpy-archetype-polyglot-app/target
9191
graalpython/graalpy-archetype-polyglot-app/target
9292
pom-mx.xml
93+
.venv

‎scripts/wheelbuilder/README.md

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,7 @@ giant one with all jobs.
2929

3030
Many packages use a lot of resources to build, and even those that do not
3131
quickly add up. We have chosen GitHub Action workflows as the cross-platform
32-
specification for how to build packages, but it is infeasible to build all
33-
packages on free GitHub Action runners. There are two ways around this:
34-
35-
### Self-hosted runners
36-
37-
The workflow files we generate declare they want to run on self-hosted runners.
38-
Anyone forking this repository to build packages can add their own runners to
39-
their fork and build packages this way. On Linux you will need Docker or Podman
40-
to use the GitHub Action runner script.
32+
specification for how to build packages.
4133

4234
### Running actions locally with nektos/act
4335

‎scripts/wheelbuilder/generate_workflow.py

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -49,10 +49,9 @@
4949
WindowsX86,
5050
)
5151

52-
BuildSpec(name="psutil")
52+
psutil = BuildSpec(name="psutil")
5353
numpy = BuildSpec(
5454
name="numpy",
55-
extra_versions=["1.21.6", "1.22.4", "1.23.1", "1.23.4"],
5655
platforms=[Linux, Mac, Windows],
5756
system_dependencies={
5857
Linux: ["gcc-toolset-12-gcc-gfortran", "openblas-devel"],
@@ -67,7 +66,7 @@
6766
pybind11 = BuildSpec(name="pybind11")
6867
ninja = BuildSpec(name="ninja")
6968
pillow = BuildSpec(
70-
name="Pillow",
69+
name="pillow",
7170
system_dependencies={
7271
Linux: [
7372
"libtiff-devel",
@@ -98,11 +97,13 @@
9897
name="scipy",
9998
spec_dependencies=[numpy],
10099
system_dependencies={
101-
Linux: ["gcc-toolset-9", "gcc-toolset-9-gcc-gfortran", "openblas-devel"],
100+
Linux: ["gcc-toolset-12-gcc-gfortran", "openblas-devel"],
102101
Mac: ["gcc", "openblas", "pkg-config"],
103102
},
104103
before_build={
105-
Linux: ["source /opt/rh/gcc-toolset-9/enable"],
104+
Linux: [
105+
"export FFLAGS=-fallow-argument-mismatch",
106+
],
106107
Mac: [
107108
"export PKG_CONFIG_PATH=/opt/homebrew/opt/openblas/lib/pkgconfig",
108109
"export FFLAGS=-fallow-argument-mismatch",
@@ -114,6 +115,9 @@
114115
spec_dependencies=[numpy, scipy],
115116
system_dependencies=["openblas"],
116117
before_build={
118+
Linux: [
119+
"export FFLAGS=-fallow-argument-mismatch",
120+
],
117121
Mac: [
118122
"export PKG_CONFIG_PATH=/opt/homebrew/opt/openblas/lib/pkgconfig",
119123
"export FFLAGS=-fallow-argument-mismatch",
@@ -122,6 +126,9 @@
122126
)
123127
cffi = BuildSpec(
124128
name="cffi",
129+
before_build=[
130+
"graalpy/bin/graalpy -m pip install wheel",
131+
],
125132
system_dependencies={
126133
Linux: ["libffi-devel"],
127134
Mac: ["libffi"],
@@ -140,42 +147,56 @@
140147
name="torch",
141148
spec_dependencies=[numpy, ninja, cmake, pybind11, cffi, pyyaml],
142149
system_dependencies={
143-
Linux: ["openblas-devel", "/usr/bin/cmake", "/usr/bin/sudo"],
144-
Mac: ["openblas", "cmake"],
150+
Linux: ["openblas-devel", "/usr/bin/cmake", "/usr/bin/sudo", "libffi-devel"],
151+
Mac: ["openblas", "cmake", "libffi"],
145152
},
146153
before_build={
154+
Linux: [
155+
"export USE_CUDA=0",
156+
],
147157
Mac: [
148158
"export USE_CUDA=0",
149159
"export PKG_CONFIG_PATH=/opt/homebrew/opt/openblas/lib/pkgconfig",
150160
],
151161
},
152-
custom_steps=[
153-
{
154-
"uses": "Jimver/[email protected]",
155-
"id": "cuda-toolkit",
156-
"if": "runner.os != 'macOS'",
157-
"with": {
158-
"cuda": "11.7.0",
159-
},
160-
}
161-
],
162162
environment={
163163
"MAX_JOBS": 4,
164164
"BUILD_TEST": 0,
165165
},
166166
)
167167
opt_einsum = BuildSpec(
168168
name="opt_einsum",
169+
spec_dependencies=[numpy],
169170
platforms=[LinuxX86],
170171
)
171172
keras_preprocessing = BuildSpec(
172-
name="keras_preprocessing",
173+
name="Keras_Preprocessing",
174+
spec_dependencies=[numpy],
175+
platforms=[LinuxX86],
176+
)
177+
grpcio = BuildSpec(
178+
name="grpcio",
179+
platforms=[LinuxX86],
180+
)
181+
ml_dtypes = BuildSpec(
182+
name="ml_dtypes",
173183
platforms=[LinuxX86],
174184
)
185+
wrapt = BuildSpec(
186+
name="wrapt",
187+
platforms=[LinuxX86],
188+
)
189+
h5py = BuildSpec(
190+
name="h5py",
191+
platforms=[LinuxX86],
192+
system_dependencies={
193+
Linux: ["hdf5-devel"],
194+
},
195+
)
175196
BuildSpec(
176197
name="tensorflow",
177198
platforms=[LinuxX86],
178-
spec_dependencies=[opt_einsum, keras_preprocessing],
199+
spec_dependencies=[grpcio, psutil, wrapt, ml_dtypes, h5py, numpy, opt_einsum, keras_preprocessing],
179200
before_build=[
180201
"export PIP_FIND_LINKS=$(pwd)",
181202
"pip install pip numpy wheel packaging requests opt_einsum",

‎scripts/wheelbuilder/wheelbuilder/__init__.py

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -71,18 +71,20 @@ class Platform:
7171
def runs_on(self) -> str:
7272
match (self.name, self.arch):
7373
case ("linux", "amd64"):
74-
return ["self-hosted", "Linux", "X64"]
75-
# return "ubuntu-latest"
74+
# return ["self-hosted", "Linux", "X64"]
75+
return "ubuntu-latest"
7676
case ("linux", "aarch64"):
7777
return ["self-hosted", "Linux", "ARM64"]
7878
# return "ubuntu-latest"
7979
case ("macos", "amd64"):
80-
# return "macos-11"
81-
return ["self-hosted", "macOS", "X64"]
80+
return "macos-12"
81+
# return ["self-hosted", "macOS", "X64"]
8282
case ("macos", "aarch64"):
83-
return ["self-hosted", "macOS", "ARM64"]
83+
return "macos-latest"
84+
# return ["self-hosted", "macOS", "ARM64"]
8485
case ("windows", "amd64"):
85-
return ["self-hosted", "windows", "X64"]
86+
# return ["self-hosted", "windows", "X64"]
87+
return "windows-latest"
8688
raise RuntimeError(f"Invalid platform spec {self.name}:{self.arch}")
8789

8890
@property
@@ -316,7 +318,7 @@ def create_jobs(
316318
steps.append(
317319
{
318320
"name": "Checkout",
319-
"uses": "actions/checkout@v3",
321+
"uses": "actions/checkout@main",
320322
}
321323
)
322324
if spec.custom_steps:
@@ -331,7 +333,7 @@ def create_jobs(
331333
steps.append(
332334
{
333335
"name": "Setup GraalPy",
334-
"uses": "actions/setup-python@main",
336+
"uses": "actions/setup-python@v5",
335337
"if": "inputs.graalpy == ''",
336338
"with": {
337339
"python-version": "graalpy24.0",
@@ -350,7 +352,7 @@ def create_jobs(
350352
steps.append(
351353
{
352354
"name": f"Download artifacts from {spec_dep.name}",
353-
"uses": "actions/download-artifact@v3",
355+
"uses": "actions/download-artifact@main",
354356
"continue-on-error": True,
355357
"with": {"name": spec_dep.platform_name(platform)},
356358
}
@@ -390,7 +392,7 @@ def create_jobs(
390392
steps.append(
391393
{
392394
"name": "Store wheels",
393-
"uses": "umutozd/upload-artifact@5c459179e7745e2c730c50b10a6459da0b6f25db",
395+
"uses": "actions/upload-artifact@main",
394396
"with": {
395397
"name": spec.platform_name(platform),
396398
"path": f"{_glob_all_from_pattern(spec.name)}*.whl",

‎scripts/wheelbuilder/wheelbuilder/generator.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -147,7 +147,7 @@ def generate(workflow_directory):
147147
+ [
148148
{
149149
"name": "Set up Python",
150-
"uses": "actions/setup-python@v4",
150+
"uses": "actions/setup-python@v5",
151151
"with": {
152152
"python-version": "3.10",
153153
},

0 commit comments

Comments
 (0)