Build platform-specific wheels containing libmagic #294
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,180 @@ | ||
| name: wheels | ||
|
|
||
| on: | ||
| pull_request: | ||
| push: | ||
| branches: master | ||
| release: | ||
| types: [released, prereleased] | ||
| workflow_dispatch: # allows running workflow manually from the Actions tab | ||
|
|
||
| concurrency: # https://stackoverflow.com/questions/66335225#comment133398800_72408109 | ||
| group: ${{ github.workflow }}-${{ github.ref || github.run_id }} | ||
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | ||
|
|
||
| jobs: | ||
| build-sdist: | ||
| runs-on: ubuntu-latest | ||
|
|
||
| env: | ||
| PIP_DISABLE_PIP_VERSION_CHECK: 1 | ||
|
|
||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 | ||
|
|
||
| - name: Set up Python | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: '3.x' | ||
|
|
||
| - run: sudo apt-get install -y libmagic1 | ||
ddelange marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| - name: Build source distribution | ||
| run: | | ||
| pip install --upgrade setuptools wheel pip build | ||
| python -m build --sdist | ||
|
|
||
| - uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: dist | ||
| path: dist/*.tar.* | ||
|
|
||
|
|
||
| build-wheels-matrix: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| include: ${{ steps.set-matrix.outputs.include }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: '3.x' | ||
| - run: pip install cibuildwheel==2.17.0 # sync version with pypa/cibuildwheel below | ||
| - id: set-matrix | ||
| env: | ||
| # only mention one (trivial) python version, as py2.py3 wheels only need to be build once per arch | ||
| CIBW_PROJECT_REQUIRES_PYTHON: '==3.12.*' | ||
| # skip PyPy wheels for now, and skip i686 wheels because pytest is failing | ||
| CIBW_SKIP: pp* *i686 | ||
| run: | | ||
| MATRIX_INCLUDE=$( | ||
| { | ||
| cibuildwheel --print-build-identifiers --platform linux --arch all | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ | ||
| && cibuildwheel --print-build-identifiers --platform macos --arch x86_64 | jq -nRc '{"only": inputs, "os": "macos-15-intel"}' \ | ||
| && cibuildwheel --print-build-identifiers --platform macos --arch arm64 | jq -nRc '{"only": inputs, "os": "macos-15"}' \ | ||
ddelange marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | jq -nRc '{"only": inputs, "os": "windows-latest"}' | ||
| } | jq -sc | ||
| ) | ||
| echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT | ||
|
|
||
|
|
||
| build-wheels: | ||
| name: build ${{ matrix.only }} | ||
| needs: build-wheels-matrix | ||
| runs-on: ${{ matrix.os }} | ||
|
|
||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }} | ||
|
|
||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 | ||
|
|
||
| - name: Set up QEMU | ||
| if: runner.os == 'Linux' | ||
| uses: docker/setup-qemu-action@v3 | ||
|
|
||
| # For Windows, grabbing latest file from MSYS2 is easier than building from source | ||
| # It's generally up to date ref https://packages.msys2.org/base/mingw-w64-file | ||
| - name: Setup MSYS2 and install file | ||
| if: runner.os == 'Windows' | ||
| uses: msys2/[email protected] | ||
| with: | ||
| msystem: ${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }} | ||
| location: D:\ | ||
| install: >- | ||
| ${{ endsWith(matrix.only, '32') && 'mingw-w64-i686-file' || 'mingw-w64-x86_64-file' }} | ||
|
|
||
| # The DLL dependency tree flattened out ref "Dependencies" https://packages.msys2.org/packages/mingw-w64-x86_64-file | ||
| - name: Copy Windows ddl and mgc | ||
| if: runner.os == 'Windows' | ||
| run: | | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/share/misc/magic.mgc" "magic" | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libmagic-1.dll" "magic" | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libsystre-0.dll" "magic" | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libtre-5.dll" "magic" | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libasprintf-0.dll" "magic" | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libintl-8.dll" "magic" | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libatomic-1.dll" "magic" | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libgomp-1.dll" "magic" | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libquadmath-0.dll" "magic" | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libstdc++-6.dll" "magic" | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libcharset-1.dll" "magic" | ||
| cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libiconv-2.dll" "magic" | ||
|
|
||
| # These are needed additionally in the win32 wheel ref https://packages.msys2.org/packages/mingw-w64-i686-file | ||
| - name: Copy additional 32-bit runtime DLLs | ||
| if: runner.os == 'Windows' && endsWith(matrix.only, '32') | ||
| run: | | ||
| cp "/msys64/mingw32/bin/libgcc_s_dw2-1.dll" "magic" | ||
| cp "/msys64/mingw32/bin/libwinpthread-1.dll" "magic" | ||
|
|
||
| - uses: pypa/[email protected] # sync version with pip install cibuildwheel above | ||
| timeout-minutes: 10 | ||
| with: | ||
| only: ${{ matrix.only }} | ||
| env: | ||
| CIBW_BUILD_VERBOSITY: 1 | ||
| # add compiled libmagic to the build directory (to include in the wheel) | ||
| CIBW_BEFORE_BUILD_MACOS: sudo -E bash add_libmagic.sh | ||
| CIBW_BEFORE_BUILD_LINUX: bash add_libmagic.sh | ||
ddelange marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| # build macos wheels with maximum backwards compatibility (gcc -mmacosx-version-min flag) | ||
| MACOSX_DEPLOYMENT_TARGET: ${{ ( endsWith( matrix.only, 'arm64' ) && '11.0' ) || '10.9' }} | ||
| # simple smoke test run on each wheel: this is an HLS MP4 video, only recognised in recent versions of libmagic | ||
| CIBW_TEST_COMMAND: python -c "import magic; assert magic.Magic(mime=True).from_buffer(b'\x00\x00\x00\x1cftypiso5\x00\x00\x00\x01isomiso5hlsf\x00\x00') == 'video/mp4'" | ||
|
|
||
| - uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: dist-${{ matrix.only }} | ||
| path: wheelhouse/*.whl | ||
|
|
||
|
|
||
| publish: | ||
| if: github.event_name == 'release' | ||
| needs: [build-sdist, build-wheels] | ||
| runs-on: ubuntu-latest | ||
|
|
||
| permissions: | ||
| contents: write # softprops/action-gh-release | ||
| id-token: write # pypa/gh-action-pypi-publish | ||
|
|
||
| steps: | ||
| - uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: 3.x | ||
|
|
||
| - uses: actions/download-artifact@v4 | ||
| with: | ||
| path: dist/ | ||
| pattern: dist-* | ||
| merge-multiple: true | ||
|
|
||
| - run: ls -ltra dist/ | ||
|
|
||
| - run: pip install --upgrade python-magic --find-links ./dist | ||
|
|
||
| - name: Smoketest | ||
| run: python -c "import magic; magic.Magic()" | ||
|
|
||
| - name: Upload release assets | ||
| uses: softprops/[email protected] | ||
| with: | ||
| files: dist/* | ||
|
|
||
| - name: Publish package distributions to PyPI | ||
| uses: pypa/[email protected] | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -euxo pipefail | ||
|
|
||
| install_source() { | ||
| # install from source | ||
| # https://www.darwinsys.com/file/ | ||
| # https://github.com/file/file/blob/FILE5_46/INSTALL#L51 | ||
| ( | ||
| version="file-5.46" && | ||
| tmpfile="$(mktemp)" && | ||
| curl -sSLo "${tmpfile}" "https://astron.com/pub/file/${version}.tar.gz" && | ||
|
There was a problem hiding this comment. Tried it on ubuntu's Docker, fails here :)
There was a problem hiding this comment. yes, the script is only intended for github actions default runner images and the cibuildwheel docker images (both have curl and make) There was a problem hiding this comment.
|
||
| tar xvf "${tmpfile}" && | ||
|
Comment on lines
+11
to
+13
This comment was marked as resolved.
Sorry, something went wrong. There was a problem hiding this comment. Not a big deal, but if going with the piped |
||
| cd "${version}" && | ||
| ./configure && | ||
| make && | ||
|
There was a problem hiding this comment. Dev dependencies to use "make" and similar commands are not shipped by default in some distros, especially in those that are usually used by Docker. I'd suggest to ensure that necessary dependencies are installed first. There was a problem hiding this comment. I wished now, it simply falls back to if make is not available, in the current setup we will only have done an unnecessary curl. not the worst but could be avoided. any ideas? There was a problem hiding this comment.
It appears to be doing so already. Try this (and also the same script, but remove the first line or replace set -euxo pipefail
func(){
echo 1
(exit 1)
echo 2
}
funcYou made me think about other topic when you mentioned that install_stuff(){
real_install()
rm potential_zip
} |
||
| make install && | ||
| make installcheck && | ||
| cd .. && | ||
| rm -r "${version}" | ||
| ) || ( cd .. && false ) | ||
| } | ||
|
|
||
| install_precompiled() { | ||
| # Mac https://formulae.brew.sh/formula/libmagic | ||
| # Debian https://packages.ubuntu.com/libmagic1 | ||
| # Alpine https://pkgs.alpinelinux.org/package/libmagic | ||
| # RHEL https://git.almalinux.org/rpms/file | ||
| if [ -n "$(which brew)" ]; then | ||
| brew install libmagic | ||
| elif [ -n "$(which apt-get)" ]; then | ||
| apt-get update | ||
| apt-get install -y libmagic1 | ||
| elif [ -n "$(which apk)" ]; then | ||
| apk add --update libmagic | ||
| elif [ -n "$(which dnf)" ]; then | ||
| dnf --setopt install_weak_deps=false -y install file-libs | ||
| fi | ||
| } | ||
|
|
||
| copy_libmagic() { | ||
| # on cibuildwheel, the lib needs to exist in the project before running setup.py | ||
| # copy lib into the magic dir, regardless of platform | ||
| # this python command relies on current working directory containing `./magic/loader.py` | ||
| libmagic_path="$(python -c 'from magic.loader import load_lib; print(load_lib()._name)')" && | ||
| cp "${libmagic_path}" "magic" && | ||
| # additionally copy compiled db into magic dir (prefer the one installed by install_source) | ||
| ( ( cp "/usr/local/share/misc/magic.mgc" "magic" || cp "/usr/share/misc/magic.mgc" "magic" ) || true ) && | ||
| # check what was copied | ||
| ls -ltra magic | ||
| } | ||
|
|
||
| # skip windows (taken care of separately in wheels.yml) | ||
| python -c 'import platform; assert platform.system() != "Windows"' || ( echo "skipping on windows" && exit 0 ) | ||
| # prefer a recent build from source | ||
| install_source || install_precompiled | ||
| # files to be copied into the wheel | ||
| copy_libmagic | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.