wheel-build.yaml

name: DeepFrame Wheels Build

permissions:
  contents: read
  id-token: write

on:
  workflow_dispatch:
    inputs:
      build_mac_arm:
        description: "Build MacOS ARM Wheels"
        type: boolean
        default: true
      build_linux_amd:
        description: "Build Linux x86_64 Wheels"
        type: boolean
        default: true
      build_linux_arm:
        description: "Build Linux ARM Wheels"
        type: boolean
        default: true
      runner:
        description: Choose MacOS Action Runner Type
        type: choice
        options:
          - "self-hosted"
          - "macos-12"
          - "macos-13"
          - "macos-14"
          - "macos-15"
        default: "self-hosted"

env:
  VCPKG_ROOT: ${{ github.workspace }}/vcpkg
  VCPKG_FEATURE_FLAGS: dependencygraph
  VCPKG_DISABLE_METRICS: true
  BUILD_PRESET: "release"

jobs:
  get_macos_matrix:
    name: Set MacOS Build Matrix
    if: ${{ inputs.build_mac_arm }}
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: set matrix
        id: set-matrix
        shell: python
        run: |-
          import os
          import json
          runner_types = []
          if "${{ inputs.build_mac_arm }}" == "true":
            runner_types.append("xlarge")
          python_to_venv = {
              '3.9' : ['9'],
              '3.10': ['10'],
              '3.11': ['11'],
              '3.12': ['12'],
              '3.13': ['13']
          }
          matrix = [{"python": py, "venv": venv, "type": type}
                    for py, venvs in python_to_venv.items()
                    for venv in venvs
                    for type in runner_types]
          with open(os.environ.get("GITHUB_OUTPUT"), "a") as file:
            file.write(f"matrix={json.dumps(matrix)}\n")
          print(f"[INFO] matrix={json.dumps(matrix)}\n")

  build_wheels-macos:
    name: MacOS Wheels Build
    needs: [get_macos_matrix]
    runs-on: ${{ inputs.runner }}-${{ matrix.type }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.get_macos_matrix.outputs.matrix) }}

    steps:
      - name: set platform name
        id: platform-name
        shell: bash
        run: |
          echo "platform=macosx_11_0_arm64" >> $GITHUB_OUTPUT
          echo "arch=arm64" >> $GITHUB_OUTPUT
          echo "triplet=arm64-osx" >> $GITHUB_OUTPUT

      - name: set vcpkg cache path
        id: vcpkg-cache
        shell: bash
        env:
          ARCH: ${{ steps.platform-name.outputs.arch }}
        run: |
          echo "cache_path=/Users/sre/GitHub/vcpkg-cache-deepframe-${{ steps.platform-name.outputs.triplet }}" >> $GITHUB_OUTPUT

      - name: clone repository
        uses: actions/checkout@v6.0.2

      - name: setup python ${{ matrix.python }}
        uses: actions/setup-python@v6.2.0
        if: inputs.runner != 'self-hosted'
        with:
          python-version: ${{ matrix.python }}

      - name: setup python ${{ matrix.python }} environment
        id: setup
        shell: bash
        run: |-
          python${{ matrix.python }} -m venv "venv_${{ matrix.venv }}"
          source "venv_${{ matrix.venv }}/bin/activate"
          python"${{ matrix.python }}" -m pip install -qqq -U pip setuptools build cmake==3.29.2
          deactivate
          echo -e "\n[bdist_wheel]\nplat-name = ${{ steps.platform-name.outputs.platform }}" >> setup.cfg

      - name: install vcpkg
        shell: bash
        run: git clone https://github.com/microsoft/vcpkg.git && cd vcpkg && git checkout 6f29f12e82a8293156836ad81cc9bf5af41fe836 && ./bootstrap-vcpkg.sh

      - name: install vcpkg dependencies
        shell: bash
        env:
          VCPKG_ROOT: ${{ github.workspace }}/vcpkg
          VCPKG_TARGET_TRIPLET: ${{ steps.platform-name.outputs.triplet }}
          VCPKG_HOST_TRIPLET: ${{ steps.platform-name.outputs.triplet }}
        run: |
          cd vcpkg
          ./vcpkg install --triplet=${{ steps.platform-name.outputs.triplet }} openssl liblzma libiconv zlib
          cd ..

      - name: clean build artifacts
        shell: bash
        run: |
          # Clean build directory to ensure fresh build
          rm -rf build
          # Clean architecture-specific FFmpeg installations
          rm -rf .ext-*

      - name: build py${{ matrix.python }}
        env:
          CMAKE_OSX_ARCHITECTURES: ${{ steps.platform-name.outputs.arch }}
          VCPKG_TARGET_TRIPLET: ${{ steps.platform-name.outputs.triplet }}
          VCPKG_HOST_TRIPLET: ${{ steps.platform-name.outputs.triplet }}
          VCPKG_BINARY_SOURCES: "clear"
          VCPKG_DISABLE_METRICS: true
          VCPKG_OVERLAY_TRIPLETS: ""
          VCPKG_OVERLAY_PORTS: ""
        shell: bash
        run: |-
          CMAKE_PATH=$(whereis cmake | awk '{print $2}')
          CMAKE_DIR=$(dirname "$CMAKE_PATH")
          echo "$CMAKE_DIR" >> $GITHUB_PATH
          source "venv_${{ matrix.venv }}/bin/activate"
          python"${{ matrix.python }}" -m pip install -r requirements-dev.txt &&
          python"${{ matrix.python }}" scripts/build.py "${{ env.BUILD_PRESET }}"
          python"${{ matrix.python }}" -m build
          deactivate

      - name: run tests
        if: matrix.type != 'x86_64'
        shell: bash
        run: |-
          source "venv_${{ matrix.venv }}/bin/activate"
          timeout -k 10s 30m python"${{ matrix.python }}" -m pytest
          deactivate

      - uses: actions/upload-artifact@v7.0.1
        if: ${{ always() }}
        with:
          name: "deepframe-macos-py${{ matrix.python }}-${{ matrix.type }}-wheels"
          path: "dist/*.whl"

  get_linux_matrix:
    name: Set Linux Build Matrix
    if: ${{ inputs.build_linux_arm || inputs.build_linux_amd }}
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: set matrix
        id: set-matrix
        shell: python
        run: |-
          import os
          import json
          runner_types = []
          if "${{ inputs.build_linux_amd }}" == "true":
            runner_types.append("x86_64")
          if "${{ inputs.build_linux_arm }}" == "true":
            runner_types.append("aarch64")
          python_to_venv = {
              '3.9' : ['9'],
              '3.10': ['10'],
              '3.11': ['11'],
              '3.12': ['12'],
              '3.13': ['13']
          }
          matrix = [{"python": py, "venv": venv, "type": type}
                    for py, venvs in python_to_venv.items()
                    for venv in venvs
                    for type in runner_types]
          with open(os.environ.get("GITHUB_OUTPUT"), "a") as file:
            file.write(f"matrix={json.dumps(matrix)}\n")
          print(f"[INFO] matrix={json.dumps(matrix)}\n")

  build_wheels_linux:
    name: Linux Wheels Build
    needs: [get_linux_matrix]
    runs-on: ${{ matrix.type == 'x86_64' && 'ubuntu-24.04' || 'ubuntu-24.04-arm' }}
    container: quay.io/activeloop/gcc-pypa-manylinux:2_28-linux-${{ matrix.type == 'x86_64' && 'amd64' || 'arm64' }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.get_linux_matrix.outputs.matrix) }}
    env:
      VCPKG_BINARY_SOURCES: "clear;files,/vcpkg-cache,readwrite"
    steps:
      - name: clone repository
        shell: bash
        run: git clone -b "${{ github.ref_name }}" "https://activeloop-bot:${{ secrets.ORG_GH_BOT_PAT }}@github.com/${{ github.repository }}" .

      - name: setup environment
        shell: bash
        run: |
          echo -e "\n[bdist_wheel]\nplat-name = manylinux2014_${{ matrix.type }}" >> setup.cfg

      - name: download vcpkg cache
        shell: bash
        run: source scripts/build_scripts/get_creds.sh && python3.11 scripts/build_scripts/manage_cache.py download deepframe

      - name: install vcpkg
        shell: bash
        run: git clone https://github.com/microsoft/vcpkg.git && cd vcpkg && git checkout 6f29f12e82a8293156836ad81cc9bf5af41fe836 && ./bootstrap-vcpkg.sh

      - name: build py${{ matrix.python }}
        shell: bash
        run: |-
          python"${{ matrix.python }}" -m pip install -r requirements-dev.txt &&
          python"${{ matrix.python }}" scripts/build.py "${{ env.BUILD_PRESET }}"
          python"${{ matrix.python }}" -m build

      - name: save vcpkg cache
        if: ${{ matrix.python == '3.11' }}
        shell: bash
        run: source scripts/build_scripts/get_creds.sh && python3.11 scripts/build_scripts/manage_cache.py upload deepframe

      - name: run tests
        shell: bash
        run: timeout -k 10s 30m python"${{ matrix.python }}" -m pytest

      - name: upload wheels to s3
        if: ${{ always() }}
        env:
          RUN_ID: ${{ github.run_id }}
          TYPE: ${{ matrix.type }}
          PYTHON: ${{ matrix.python }}
        shell: bash
        run: source scripts/build_scripts/get_creds.sh && python3.11 scripts/build_scripts/upload_wheels.py deepframe