diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml deleted file mode 100644 index 4b9ccd29d82..00000000000 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ /dev/null @@ -1,161 +0,0 @@ -name: Build Linux arm64 - -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' - - -jobs: - build_wheels_linux: - name: ${{ matrix.os }} py${{ matrix.python-version }} - runs-on: GH-Linux-ARM64 - strategy: - fail-fast: false - matrix: - os: [ ubuntu-24.04 ] - python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ] - # python-version: [ "3.7" ] - env: - RUNNER_OS: ${{ matrix.os }} - PYTHON_VERSION: ${{ matrix.python-version }} - steps: - - name: Install python - uses: actions/setup-python@v4 - with: - python-version: "${{ matrix.python-version }}" - - name: Install clang++ for Ubuntu - if: matrix.os == 'ubuntu-24.04' - run: | - pwd - uname -a - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 18 - which clang++-18 - clang++-18 --version - sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget - ccache -s - - name: Update git - run: | - sudo add-apt-repository ppa:git-core/ppa -y - sudo apt-get update - sudo apt-get install -y git - git --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ matrix.os }} - max-size: 5G - append-timestamp: true - - name: remove old clang and link clang-18 to clang - if: matrix.os == 'ubuntu-24.04' - run: | - sudo rm -f /usr/bin/clang || true - sudo ln -s /usr/bin/clang-18 /usr/bin/clang - sudo rm -f /usr/bin/clang++ || true - sudo ln -s /usr/bin/clang++-18 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Run chdb/build.sh - run: | - python3 -m pip install pybind11 setuptools - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - bash ./chdb/build.sh - python3 -m pip install pandas pyarrow - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Install dependencies for building wheels - run: python3 -m pip install -U pip tox pybind11 twine setuptools wheel - - name: Build wheels - run: | - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - make wheel - - name: Install patchelf from github - run: | - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - sudo cp bin/patchelf /usr/bin/ - sudo chmod +x /usr/bin/patchelf - patchelf --version - - name: Audit wheels - run: | - python3 -m pip install auditwheel - auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - sudo rm -f dist/*linux_aarch64.whl - ls -lh dist - shell: bash - - name: Run tests - run: | - python3 -m pip install dist/*.whl - python3 -m pip install pandas pyarrow psutil - python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)" - make test - continue-on-error: false - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - if: matrix.python-version == '3.12' - run: | - cp programs/local/chdb.h chdb.h - tar -czvf linux-aarch64-libchdb.tar.gz libchdb.so chdb.h - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') && matrix.python-version == '3.12' - run: | - gh release upload ${{ github.ref_name }} linux-aarch64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-linux-aarch64-${{ matrix.python-version }} - path: | - ./dist/*.whl - ./linux-aarch64-libchdb.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - python3 -m pip install twine - python3 -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - diff --git a/.github/workflows/build_linux_x86_wheels.yml b/.github/workflows/build_linux_x86_wheels.yml deleted file mode 100644 index a915d2a636e..00000000000 --- a/.github/workflows/build_linux_x86_wheels.yml +++ /dev/null @@ -1,160 +0,0 @@ -name: Build Linux X86 - -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' - - -jobs: - build_wheels_linux: - name: ${{ matrix.os }} py${{ matrix.python-version }} - runs-on: gh-64c - strategy: - fail-fast: false - matrix: - os: [ ubuntu-22.04 ] - python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ] - # python-version: [ "3.7" ] - env: - RUNNER_OS: ${{ matrix.os }} - PYTHON_VERSION: ${{ matrix.python-version }} - steps: - - name: Install python - uses: actions/setup-python@v4 - with: - python-version: "${{ matrix.python-version }}" - - name: Install clang++ for Ubuntu - if: matrix.os == 'ubuntu-22.04' - run: | - pwd - uname -a - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 18 - which clang++-18 - clang++-18 --version - sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget - ccache -s - - name: Update git - run: | - sudo add-apt-repository ppa:git-core/ppa -y - sudo apt-get update - sudo apt-get install -y git - git --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ matrix.os }} - max-size: 5G - append-timestamp: true - - name: remove old clang and link clang-18 to clang - if: matrix.os == 'ubuntu-22.04' - run: | - sudo rm -f /usr/bin/clang || true - sudo ln -s /usr/bin/clang-18 /usr/bin/clang - sudo rm -f /usr/bin/clang++ || true - sudo ln -s /usr/bin/clang++-18 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Run chdb/build.sh - run: | - python3 -m pip install pybind11 setuptools - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - bash ./chdb/build.sh - python3 -m pip install pandas pyarrow - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Install dependencies for building wheels - run: python3 -m pip install -U pip tox pybind11 twine setuptools wheel - - name: Build wheels - run: | - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - make wheel - - name: Install patchelf from github - run: | - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - sudo cp bin/patchelf /usr/bin/ - sudo chmod +x /usr/bin/patchelf - patchelf --version - - name: Audit wheels - run: | - python3 -m pip install auditwheel - auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - sudo rm -f dist/*-linux_x86_64.whl - ls -lh dist - shell: bash - - name: Run tests - run: | - python3 -m pip install dist/*.whl - python3 -m pip install pandas pyarrow psutil - python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)" - make test - continue-on-error: false - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - if: matrix.python-version == '3.12' - run: | - cp programs/local/chdb.h chdb.h - tar -czvf linux-x86_64-libchdb.tar.gz libchdb.so chdb.h - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') && matrix.python-version == '3.12' - run: | - gh release upload ${{ github.ref_name }} linux-x86_64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-linux-x86_64-${{ matrix.python-version }} - path: | - ./dist/*.whl - ./linux-x86_64-libchdb.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - python3 -m pip install twine - python3 -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml similarity index 91% rename from .github/workflows/build_macos_x86_wheels.yml rename to .github/workflows/build_macos_arm64_wheels.yml index c4473a9b0e6..4ba86ea74b2 100644 --- a/.github/workflows/build_macos_x86_wheels.yml +++ b/.github/workflows/build_macos_arm64_wheels.yml @@ -1,4 +1,4 @@ -name: Build macOS X86 +name: Build macOS arm64 on: workflow_dispatch: @@ -13,6 +13,11 @@ on: - main paths-ignore: - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' jobs: build_wheels_macos_13: @@ -21,8 +26,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ macos-13 ] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + os: [ macos-14 ] + python-version: ["3.12"] env: RUNNER_OS: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} @@ -115,7 +120,7 @@ jobs: make wheel - name: Fix wheel platform tag run: | - python3 -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl + python3 -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl - name: Run tests run: | python3 -m pip install dist/*.whl @@ -136,19 +141,19 @@ jobs: if: matrix.python-version == '3.12' run: | cp programs/local/chdb.h chdb.h - tar -czvf macos-x86_64-libchdb.tar.gz libchdb.so chdb.h + tar -czvf macos-aarch64-libchdb.tar.gz libchdb.so chdb.h - name: Upload libchdb.so to release if: startsWith(github.ref, 'refs/tags/v') && matrix.python-version == '3.12' run: | - gh release upload ${{ github.ref_name }} macos-x86_64-libchdb.tar.gz --clobber + gh release upload ${{ github.ref_name }} macos-aarch64-libchdb.tar.gz --clobber env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - uses: actions/upload-artifact@v4 with: - name: chdb-artifacts-macos-x86_64-${{ matrix.python-version }} + name: chdb-artifacts-macos-aarch64-${{ matrix.python-version }} path: | ./dist/*.whl - ./macos-x86_64-libchdb.tar.gz + ./macos-aarch64-libchdb.tar.gz overwrite: true - name: Upload pypi if: startsWith(github.ref, 'refs/tags/v') @@ -158,4 +163,3 @@ jobs: env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - diff --git a/chdb/build.sh b/chdb/build.sh index c0cd879763c..8a087333b51 100755 --- a/chdb/build.sh +++ b/chdb/build.sh @@ -15,9 +15,12 @@ HDFS="-DENABLE_HDFS=1 -DENABLE_GSASL_LIBRARY=1 -DENABLE_KRB5=1" MYSQL="-DENABLE_MYSQL=1" # check current os type if [ "$(uname)" == "Darwin" ]; then - export CXX=/usr/local/opt/llvm/bin/clang++ - export CC=/usr/local/opt/llvm/bin/clang - export PATH=/usr/local/opt/llvm/bin:$PATH + # export CXX=/usr/local/opt/llvm/bin/clang++ + # export CC=/usr/local/opt/llvm/bin/clang + # export PATH=/usr/local/opt/llvm/bin:$PATH + export CXX=/opt/homebrew/opt/llvm@18/bin/clang++ + export CC=/opt/homebrew/opt/llvm@18/bin/clang + export PATH="/opt/homebrew/opt/llvm@18/bin:$PATH" GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=0" UNWIND="-DUSE_UNWIND=0" JEMALLOC="-DENABLE_JEMALLOC=0" @@ -28,7 +31,7 @@ if [ "$(uname)" == "Darwin" ]; then SED_INPLACE="sed -i ''" # if Darwin ARM64 (M1, M2), disable AVX if [ "$(uname -m)" == "arm64" ]; then - CMAKE_TOOLCHAIN_FILE="-DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-aarch64.cmake" + # CMAKE_TOOLCHAIN_FILE="-DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-aarch64.cmake" CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0" LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0" else @@ -95,7 +98,6 @@ CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 -DENABLE_CLICKHOUSE_ALL=0 -DUSE_STATIC_LIBRARIES=1 -DSPLIT_SHARED_LIBRARIES=0 \ -DENABLE_SIMDJSON=1 -DENABLE_RAPIDJSON=1 \ ${CPU_FEATURES} \ - ${CMAKE_TOOLCHAIN_FILE} \ -DENABLE_AVX512=0 -DENABLE_AVX512_VBMI=0 \ -DENABLE_LIBFIU=1 \ -DCHDB_VERSION=${CHDB_VERSION} \