build(ci): Enable hosted GPU Spark compile checks

.github/scripts/cmake-flags.sh

@@ -25,6 +25,8 @@
 #   * dep-graph            — Dep-graph generator action; same as adapters minus
 #                            WAVE/CUDF (they gate experimental code, which the
 #                            selective-build planner short-circuits to full).
+#   * gpu-spark-hosted     — Minimal cuDF/UCX production-object compile on a
+#                            standard GitHub-hosted runner.
 #   * ubuntu-debug         — Ubuntu debug with system dependencies.
 #   * ubuntu-bundled-deps  — Ubuntu debug with velox's BUNDLED dependency
 #                            resolution; mirrors ubuntu-debug minus
@@ -43,6 +45,14 @@
 #   EXTRA_CMAKE_FLAGS=("${CMAKE_FLAGS[@]}" -DVELOX_ENABLE_X=ON)
 
 case "${BUILD_PROFILE:?BUILD_PROFILE must be set before sourcing cmake-flags.sh}" in
+gpu-spark-hosted)
+  CMAKE_FLAGS=(
+    -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+    -DVELOX_MONO_LIBRARY=OFF
+    -DVELOX_ENABLE_CUDF=ON
+  )
+  ;;
+
 adapters | dep-graph)
   # adapters and dep-graph are sister profiles: dep-graph regenerates
   # the planner's source-of-truth graph for what adapters builds, so

.github/workflows/README.md

@@ -18,12 +18,13 @@ For current build times and performance trends, see the [CI performance metrics]
 
 | Workflow | File | Triggers | Purpose |
 |----------|------|----------|---------|
+| GPU Spark Hosted Compile | `gpu-spark-hosted-compile.yml` | push to dev, PRs, manual | Compile cuDF and UCX production objects on a standard GitHub-hosted runner without requiring a GPU |
 | Linux Build using GCC | `linux-build.yml` + `linux-build-base.yml` | push to main, PRs | Main build & test (3 configs); selective by default, full on push / sticky-approval |
 | Detect Force-Full Trigger | `detect-force-full.yml` | PR approving review | Step 1 of the approval-path chain: trivial trigger for the workflow_run hand-off (needed because fork PR reviews get a read-only token) |
 | Rerun Linux Build on Force-Full Trigger | `rerun-on-force-full.yml` | workflow_run (after Detect Force-Full Trigger) | Step 2 of the approval-path chain: BASE-context, dedups, calls the `rerun-linux-build` composite action |
 | Force-Full Build on /full-build Comment | `force-full-on-comment.yml` | PR comment containing `/full-build` | Direct (no workflow_run hop): dedups, calls the `rerun-linux-build` composite action |
 | Selective Build Comment | `selective-build-comment.yml` | workflow_run (after Linux Build) | Post selective build plan as PR comment |
-| macOS Build | `macos.yml` | push, PRs | Compilation check (debug + release) |
+| macOS Build | `macos.yml` | push to dev, PRs | Core API compilation check (debug) |
 | Breeze Linux Build | `breeze.yml` | push to main, PRs | Tracing module with sanitizers |
 | Fuzzer Jobs | `scheduled.yml` | PRs, push to main, daily cron, manual | Randomized correctness testing |
 | Run Checks | `preliminary_checks.yml` | PRs | Formatting, linting, PR title |
@@ -40,6 +41,22 @@ For current build times and performance trends, see the [CI performance metrics]
 
 ## Core Build & Test
 
+### GPU Spark Hosted Compile (`gpu-spark-hosted-compile.yml`)
+
+Uses the existing adapters CI image and a minimal cuDF CMake profile on a
+standard four-core GitHub-hosted runner. It configures the complete cuDF build
+graph without unrelated cloud adapters, then asks Ninja to compile only
+Velox-owned production objects under
+`velox/experimental/cudf` and `velox/experimental/ucx-exchange`. Building
+objects directly avoids compiling and linking all of libcudf while still
+checking C++, CUDA, generated-header, and public API compatibility.
+The current adapters image does not include UCX, so the workflow installs it
+through `scripts/setup-centos-adapters.sh` and caches the resulting prefix.
+
+This lane does not link the resulting libraries or execute GPU tests. Those
+remain responsibilities of the full adapters and `cudf-tests` jobs on larger
+and GPU-backed runners.
+
 ### Linux Build using GCC (`linux-build.yml` + `linux-build-base.yml`)
 
 The main CI workflow for Velox. Triggered on pushes to `main` and on pull request `opened`/`synchronize`/`reopened` events. The entry point `linux-build.yml` first calls the `selective-build-plan.yml` reusable workflow to decide the build mode (see [Selective Build Plan](#selective-build-plan-selective-build-planyml)), then delegates to `linux-build-base.yml` which can build and test up to four configurations in parallel on 32-core Ubuntu runners:
@@ -65,7 +82,16 @@ Status jobs handle cancelled runs gracefully — when a job is cancelled (e.g.,
 
 ### macOS Build (`macos.yml`)
 
-Builds Velox on macOS 15 (ARM64/Apple Silicon) with both debug and release configurations. Triggered on pushes to any branch and on pull requests when relevant files change. Uses the Ninja build generator and ccache for faster builds. Tests are currently disabled on macOS — the workflow focuses on ensuring compilation compatibility with Apple's toolchain rather than full test coverage. Dependencies are installed via the `setup-macos.sh` script.
+Builds the `velox_core_test` target in the debug configuration on macOS 15
+(ARM64/Apple Silicon) for pushes and pull requests targeting `dev`. This target
+compiles and links the shared core API surface and its production dependencies
+without spending hosted-runner time on every unrelated test binary. Pure cuDF
+and UCX changes are excluded because those modules are not enabled by the macOS
+CMake profile; changes to shared Velox code still receive the cross-platform
+compile check. Uses the Ninja build generator and ccache for faster builds.
+Tests are currently disabled on macOS — the workflow focuses on ensuring
+compilation compatibility with Apple's toolchain rather than full test
+coverage. Dependencies are installed via the `setup-macos.sh` script.
 
 ### Breeze Linux Build (`breeze.yml`)
 

.github/workflows/gpu-spark-hosted-compile.yml

@@ -0,0 +1,186 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: GPU Spark Hosted Compile
+
+on:
+  workflow_dispatch: {}
+  push:
+    branches:
+      - dev
+    paths:
+      - velox/experimental/cudf/**
+      - velox/experimental/ucx-exchange/**
+      - velox/functions/sparksql/**
+      - velox/common/**
+      - velox/core/**
+      - velox/exec/**
+      - velox/vector/**
+      - CMakeLists.txt
+      - CMake/**
+      - Makefile
+      - scripts/setup-centos-adapters.sh
+      - scripts/setup-centos9.sh
+      - scripts/setup-common.sh
+      - scripts/setup-helper-functions.sh
+      - scripts/setup-versions.sh
+      - .github/scripts/cmake-flags.sh
+      - .github/workflows/gpu-spark-hosted-compile.yml
+  pull_request:
+    branches:
+      - dev
+    paths:
+      - velox/experimental/cudf/**
+      - velox/experimental/ucx-exchange/**
+      - velox/functions/sparksql/**
+      - velox/common/**
+      - velox/core/**
+      - velox/exec/**
+      - velox/vector/**
+      - CMakeLists.txt
+      - CMake/**
+      - Makefile
+      - scripts/setup-centos-adapters.sh
+      - scripts/setup-centos9.sh
+      - scripts/setup-common.sh
+      - scripts/setup-helper-functions.sh
+      - scripts/setup-versions.sh
+      - .github/scripts/cmake-flags.sh
+      - .github/workflows/gpu-spark-hosted-compile.yml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  compile:
+    name: cuDF and UCX object compile
+    runs-on: ubuntu-22.04
+    timeout-minutes: 90
+    env:
+      BUILD_IMAGE: ghcr.io/facebookincubator/velox-dev@sha256:480989697666235f5d5c4ea7cdb26145b9722fc6db6e0cc2fe0bcfa369ffa199
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Free disk space
+        run: |
+          df -h /
+          sudo rm -rf \
+            /usr/share/dotnet \
+            /usr/local/lib/android \
+            /opt/ghc \
+            /usr/local/.ghcup \
+            /opt/hostedtoolcache/CodeQL
+          df -h /
+
+      - name: Restore UCX
+        id: ucx-cache
+        uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        with:
+          path: hosted-deps/ucx
+          key: hosted-ucx-${{ runner.os }}-1.20.1-v1
+
+      - name: Restore compiler cache
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        with:
+          path: ccache
+          key: hosted-gpu-spark-${{ runner.os }}-${{ hashFiles('CMake/resolve_dependency_modules/cudf.cmake', '.github/scripts/cmake-flags.sh') }}-${{ github.sha }}
+          restore-keys: |
+            hosted-gpu-spark-${{ runner.os }}-
+
+      - name: Pull build image
+        run: |
+          docker pull "$BUILD_IMAGE"
+
+      - name: Build UCX dependency
+        if: steps.ucx-cache.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p hosted-deps/ucx
+          docker run --rm \
+            --entrypoint /bin/bash \
+            --env BUILD_THREADS=4 \
+            --env DEPENDENCY_DIR=/tmp/velox-hosted-deps \
+            --env INSTALL_PREFIX=/workspace/hosted-deps/ucx \
+            --env UCX_VERSION=1.20.1 \
+            --mount "type=bind,source=$GITHUB_WORKSPACE,target=/workspace" \
+            --workdir /workspace \
+            "$BUILD_IMAGE" -lc '
+              set -euo pipefail
+              mkdir -p "$DEPENDENCY_DIR"
+              source scripts/setup-centos-adapters.sh
+              install_ucx
+            '
+
+      - name: Save UCX
+        if: steps.ucx-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        with:
+          path: hosted-deps/ucx
+          key: hosted-ucx-${{ runner.os }}-1.20.1-v1
+
+      - name: Configure and compile GPU Spark objects
+        run: |
+          mkdir -p ccache
+          docker run --rm \
+            --entrypoint /bin/bash \
+            --env CCACHE_DIR=/workspace/ccache \
+            --env CCACHE_MAXSIZE=2G \
+            --env CCACHE_NOHASHDIR=true \
+            --env CMAKE_PREFIX_PATH=/workspace/hosted-deps/ucx \
+            --env CPATH=/workspace/hosted-deps/ucx/include \
+            --env LD_LIBRARY_PATH=/workspace/hosted-deps/ucx/lib:/workspace/hosted-deps/ucx/lib64 \
+            --env LIBRARY_PATH=/workspace/hosted-deps/ucx/lib:/workspace/hosted-deps/ucx/lib64 \
+            --env PKG_CONFIG_PATH=/workspace/hosted-deps/ucx/lib/pkgconfig:/workspace/hosted-deps/ucx/lib64/pkgconfig \
+            --env VELOX_DEPENDENCY_SOURCE=SYSTEM \
+            --env cudf_SOURCE=BUNDLED \
+            --mount "type=bind,source=$GITHUB_WORKSPACE,target=/workspace" \
+            --workdir /workspace \
+            "$BUILD_IMAGE" -lc '
+              set -euo pipefail
+              git config --global --add safe.directory /workspace
+              source /opt/rh/gcc-toolset-14/enable
+              dnf -y install rdma-core-devel
+
+              BUILD_PROFILE=gpu-spark-hosted source .github/scripts/cmake-flags.sh
+              export CUDA_ARCHITECTURES=70
+              export CUDA_COMPILER=/usr/local/cuda-12.9/bin/nvcc
+              export CUDA_FLAGS="-ccbin /opt/rh/gcc-toolset-14/root/usr/bin"
+
+              CC=gcc CXX=g++ make cmake \
+                BUILD_DIR=hosted-gpu-spark \
+                BUILD_TYPE=Release \
+                VELOX_BUILD_TESTING=OFF \
+                MAX_HIGH_MEM_JOBS=2 \
+                MAX_LINK_JOBS=1 \
+                EXTRA_CMAKE_FLAGS="${CMAKE_FLAGS[*]}"
+
+              mapfile -t objects < <(
+                ninja -C _build/hosted-gpu-spark -t targets all |
+                  sed -nE "s#^(velox/experimental/(cudf|ucx-exchange)/[^:]+\\.o):.*#\\1#p"
+              )
+              if (( ${#objects[@]} == 0 )); then
+                echo "::error::CMake generated no cuDF or UCX object targets."
+                exit 1
+              fi
+
+              printf "Compiling %d Velox GPU object targets.\n" "${#objects[@]}"
+              printf "%s\n" "${objects[@]}"
+              ninja -C _build/hosted-gpu-spark -j 4 "${objects[@]}"
+              ccache -s
+            '

.github/workflows/macos.yml

@@ -17,9 +17,29 @@ name: macOS Build
 on:
   workflow_dispatch: {}
   push:
+    branches:
+      - dev
     paths:
       - velox/**
       - '!velox/docs/**'
+      - '!velox/experimental/cudf/**'
+      - '!velox/experimental/ucx-exchange/**'
+      - CMakeLists.txt
+      - CMake/**
+      - scripts/setup-macos.sh
+      - scripts/setup-common.sh
+      - scripts/setup-versions.sh
+      - scripts/setup-helper-functions.sh
+      - .github/workflows/macos.yml
+
+  pull_request:
+    branches:
+      - dev
+    paths:
+      - velox/**
+      - '!velox/docs/**'
+      - '!velox/experimental/cudf/**'
+      - '!velox/experimental/ucx-exchange/**'
       - CMakeLists.txt
       - CMake/**
       - scripts/setup-macos.sh
@@ -37,13 +57,13 @@ concurrency:
 
 jobs:
   macos-build:
-    if: ${{ github.repository == 'facebookincubator/velox' }}
     name: macos-15-${{ matrix.type }}
+    timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
         # macos-15 = arm64 Mac and cmake 4.0 with 7GB RAM
-        type: [debug, release]
+        type: [debug]
     runs-on: macos-15
     env:
       CCACHE_DIR: ${{ github.workspace }}/ccache
@@ -104,12 +124,13 @@ jobs:
               "${CMAKE_FLAGS[@]}" \
               -DCMAKE_BUILD_TYPE=$BUILD_TYPE
 
-      - name: Build
+      - name: Build core API surface
         run: |
-          cmake --build _build/$BUILD_TYPE -j $NJOBS
+          cmake --build _build/$BUILD_TYPE --target velox_core_test -j $NJOBS
           ccache -s
 
       - uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4
+        if: ${{ !cancelled() }}
         with:
           path: ${{ env.CCACHE_DIR }}
           key: ccache-macos-1-macos-15-${{ matrix.type }}