WIP

timhurskidremio · timhurskidremio · commit 1f962d37c449 · 2026-02-25T12:04:53.000-06:00
diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml
@@ -228,12 +228,13 @@ jobs:
         with:
           repository: Microsoft/vcpkg
           path: arrow/vcpkg
+          fetch-depth: 0
       - name: Install vcpkg
         run: |
           cd arrow/vcpkg
           ./bootstrap-vcpkg.sh
-          echo "VCPKG_ROOT=${PWD}/arrow/vcpkg" >> ${GITHUB_ENV}
-          echo "${PWD}/arrow/vcpkg" >> ${GITHUB_PATH}
+          echo "VCPKG_ROOT_LOCAL=${PWD}" >> ${GITHUB_ENV}
+          echo "${PWD}" >> ${GITHUB_PATH}
       - name: Clean up disk space
         run: |
           echo "=== Free disk space before cleanup ==="
@@ -284,25 +285,28 @@ jobs:
           brew bundle --file=arrow/cpp/Brewfile
 
           # Clean up any existing LLVM installations in favor of vcpkg.
-          brew uninstall llvm || :
+          # Need to uninstall all versioned LLVM packages (llvm@18, llvm@17, etc.)
+          for llvm_pkg in $(brew list | grep -E '^llvm(@[0-9]+)?$'); do
+            brew uninstall "${llvm_pkg}" || :
+          done
 
           # We want to link aws-sdk-cpp statically but Homebrew's
           # aws-sdk-cpp provides only shared library. If we have
           # Homebrew's aws-sdk-cpp, our build mix Homebrew's
           # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
           # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
-          brew uninstall aws-sdk-cpp
+          brew uninstall aws-sdk-cpp || :
           # We want to use bundled RE2 for static linking. If
           # Homebrew's RE2 is installed, its header file may be used.
           # We uninstall Homebrew's RE2 to ensure using bundled RE2.
           brew uninstall grpc || : # gRPC depends on RE2
           brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too
-          brew uninstall re2
+          brew uninstall re2 || :
           # We want to use bundled Protobuf for static linking. If
           # Homebrew's Protobuf is installed, its library file may be
           # used on test  We uninstall Homebrew's Protobuf to ensure using
           # bundled Protobuf.
-          brew uninstall protobuf
+          brew uninstall protobuf || :
 
           echo ""
           echo "=== Free disk space before LLVM build ==="
@@ -312,7 +316,8 @@ jobs:
           # Use vcpkg to install LLVM.
           vcpkg install \
             --clean-after-build \
-            --x-install-root=${VCPKG_ROOT}/installed \
+            --vcpkg-root=${VCPKG_ROOT_LOCAL} \
+            --x-install-root=${VCPKG_ROOT_LOCAL}/installed \
             --x-manifest-root=arrow/ci/vcpkg \
             --overlay-ports=arrow/ci/vcpkg/overlay/llvm/ \
             --x-feature=gandiva-llvm
@@ -558,7 +563,7 @@ jobs:
           # GH-499: How to create release notes?
           echo "Creating release: ${{ steps.commit_ids.outputs.release_tag }}"
           gh release create "${{ steps.commit_ids.outputs.release_tag }}" \
-            -n "Action URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
+            -n "Release ${{ steps.commit_ids.outputs.release_name }} RC${{ steps.commit_ids.outputs.rc }}<br>Triggered by: ${{ github.actor }}<br>Action URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID<br>arrow_branch: ${{github.event.inputs.ARROW_BRANCH}}<br>arrow_repo: ${{github.event.inputs.ARROW_REPO}}<br>release_tag_name: ${{github.event.inputs.RELEASE_TAG_NAME}}<br>arrow-java branch: ${{github.ref_name}}" \
             --prerelease \
             --repo ${GITHUB_REPOSITORY} \
             --title "Apache Arrow Java ${{ steps.commit_ids.outputs.version }} RC${{ steps.commit_ids.outputs.rc }} (arrow-java: ${{ steps.commit_ids.outputs.arrow_java_commit }}, arrow: ${{ steps.commit_ids.outputs.arrow_commit }})"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -25,6 +25,7 @@ repos:
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
+      - id: check-shebang-scripts-are-executable
       - id: check-yaml
       - id: check-added-large-files
       - id: file-contents-sorter
@@ -43,8 +44,8 @@ repos:
               rm -f apache-arrow-java.tar.gz"
         always_run: true
         pass_filenames: false
-  - repo: https://github.com/koalaman/shellcheck-precommit
-    rev: v0.10.0
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.10.0.1
     hooks:
       - id: shellcheck
         args:
diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh
@@ -61,7 +61,7 @@ github_actions_group_begin "Building Arrow C++ libraries"
 install_dir="${build_dir}/cpp-install"
 : "${ARROW_ACERO:=ON}"
 export ARROW_ACERO
-: "${ARROW_BUILD_TESTS:=ON}"
+: "${ARROW_BUILD_TESTS:=OFF}"
 export ARROW_BUILD_TESTS
 : "${ARROW_DATASET:=ON}"
 export ARROW_DATASET
@@ -78,6 +78,55 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
 
+# Determine vcpkg triplet based on architecture
+vcpkg_arch="$(arch)"
+case "${vcpkg_arch}" in
+arm64)
+  vcpkg_triplet="arm64-osx"
+  ;;
+i386 | x86_64)
+  vcpkg_triplet="x64-osx"
+  ;;
+*)
+  vcpkg_triplet="arm64-osx"
+  ;;
+esac
+
+# Set LLVM_DIR to point to vcpkg-installed LLVM if VCPKG_ROOT_LOCAL is set
+llvm_dir_arg=""
+gandiva_cxx_flags=""
+osx_sysroot_arg=""
+re2_source_arg="-Dre2_SOURCE=BUNDLED"
+if [ -n "${VCPKG_ROOT_LOCAL:-}" ]; then
+  vcpkg_installed="${VCPKG_ROOT_LOCAL}/installed/${vcpkg_triplet}"
+  llvm_cmake_dir="${vcpkg_installed}/share/llvm"
+  if [ -d "${llvm_cmake_dir}" ]; then
+    llvm_dir_arg="-DLLVM_DIR=${llvm_cmake_dir}"
+
+    # vcpkg's clang needs to know where to find system headers
+    # Arrow's GandivaAddBitcode.cmake uses CMAKE_OSX_SYSROOT to set SDKROOT env var
+    sdk_path="$(xcrun --show-sdk-path)"
+    if [ -d "${sdk_path}" ]; then
+      osx_sysroot_arg="-DCMAKE_OSX_SYSROOT=${sdk_path}"
+    fi
+
+    # Also pass the C++ standard library include path via ARROW_GANDIVA_PC_CXX_FLAGS
+    xcode_path="$(xcode-select -p)"
+    cxx_include_path="${xcode_path}/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1"
+    if [ -d "${cxx_include_path}" ]; then
+      gandiva_cxx_flags="-DARROW_GANDIVA_PC_CXX_FLAGS=-stdlib=libc++;-isystem;${cxx_include_path}"
+    fi
+
+    # Use vcpkg's RE2 since it's installed as a dependency of LLVM
+    # This ensures ABI compatibility - vcpkg's RE2 uses std::string_view API
+    # which matches what vcpkg's LLVM and Abseil expect
+    re2_cmake_dir="${vcpkg_installed}/share/re2"
+    if [ -d "${re2_cmake_dir}" ]; then
+      re2_source_arg="-Dre2_ROOT=${vcpkg_installed}"
+    fi
+  fi
+fi
+
 cmake \
   -S "${arrow_dir}/cpp" \
   -B "${build_dir}/cpp" \
@@ -100,10 +149,13 @@ cmake \
   -DCMAKE_INSTALL_PREFIX="${install_dir}" \
   -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \
   -DGTest_SOURCE=BUNDLED \
+  "${llvm_dir_arg}" \
+  "${osx_sysroot_arg}" \
+  "${gandiva_cxx_flags}" \
   -DPARQUET_BUILD_EXAMPLES=OFF \
   -DPARQUET_BUILD_EXECUTABLES=OFF \
   -DPARQUET_REQUIRE_ENCRYPTION=OFF \
-  -Dre2_SOURCE=BUNDLED \
+  "${re2_source_arg}" \
   -GNinja
 cmake --build "${build_dir}/cpp" --target install
 github_actions_group_end
@@ -125,7 +177,27 @@ if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then
   github_actions_group_end
 fi
 
-export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install"
+# Pass paths to dependencies so the JNI build can find them
+# Build up the JNI CMake args based on what's available
+jni_cmake_args="${llvm_dir_arg}"
+
+# Add Protobuf path if bundled, otherwise CMake will find system Protobuf
+if [ -d "${build_dir}/cpp/protobuf_ep-install" ]; then
+  jni_cmake_args="${jni_cmake_args} -DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install"
+fi
+
+# RE2 path for the JNI build - prefer vcpkg's RE2 if we used it for the C++ build,
+# otherwise fall back to bundled RE2 if available
+if [ -n "${VCPKG_ROOT_LOCAL:-}" ]; then
+  vcpkg_re2_dir="${VCPKG_ROOT_LOCAL}/installed/${vcpkg_triplet}"
+  if [ -d "${vcpkg_re2_dir}/share/re2" ]; then
+    jni_cmake_args="${jni_cmake_args} -Dre2_ROOT=${vcpkg_re2_dir}"
+  fi
+elif [ -d "${build_dir}/cpp/re2_ep-install" ]; then
+  jni_cmake_args="${jni_cmake_args} -Dre2_ROOT=${build_dir}/cpp/re2_ep-install"
+fi
+
+export JAVA_JNI_CMAKE_ARGS="${jni_cmake_args}"
 "${source_dir}/ci/scripts/jni_build.sh" \
   "${source_dir}" \
   "${install_dir}" \
@@ -153,6 +225,7 @@ archery linking check-dependencies \
   --allow libncurses \
   --allow libobjc \
   --allow libz \
+  --allow libz3 \
   "arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.dylib" \
   "arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.dylib" \
   "arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib" \
diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh
@@ -26,19 +26,19 @@ set -euo pipefail
 . "$(dirname "${0}")/util_log.sh"
 
 github_actions_group_begin "Update llvm"
-    vcpkg install \
-        --debug \
-        --clean-after-build \
-        --x-install-root=${VCPKG_ROOT}/installed \
-        --x-manifest-root=/arrow/ci/vcpkg \
-        --overlay-ports=/arrow/ci/vcpkg/overlay/llvm/ \
-        --x-feature=dev \
-        --x-feature=flight \
-        --x-feature=gcs \
-        --x-feature=json \
-        --x-feature=parquet \
-        --x-feature=gandiva \
-        --x-feature=s3
+vcpkg install \
+  --debug \
+  --clean-after-build \
+  --x-install-root="${VCPKG_ROOT}/installed" \
+  --x-manifest-root=/arrow/ci/vcpkg \
+  --overlay-ports=/arrow/ci/vcpkg/overlay/llvm/ \
+  --x-feature=dev \
+  --x-feature=flight \
+  --x-feature=gcs \
+  --x-feature=json \
+  --x-feature=parquet \
+  --x-feature=gandiva \
+  --x-feature=s3
 github_actions_group_end
 
 github_actions_group_begin "Prepare arguments"
diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java
@@ -53,7 +53,6 @@ public class Union${listName}Writer extends AbstractFieldWriter {
   private boolean inStruct = false;
   private boolean listStarted = false;
   private String structName;
-  private ArrowType extensionType;
   <#if listName == "LargeList" || listName == "LargeListView">
   private static final long OFFSET_WIDTH = 8;
   <#else>
@@ -204,13 +203,13 @@ public MapWriter map(String name, boolean keysSorted) {
 
   @Override
   public ExtensionWriter extension(ArrowType arrowType) {
-    extensionType = arrowType;
-    return this;
+    writer.extension(arrowType);
+    return writer;
   }
-
   @Override
   public ExtensionWriter extension(String name, ArrowType arrowType) {
-    return writer.extension(name, arrowType);
+    ExtensionWriter extensionWriter = writer.extension(name, arrowType);
+    return extensionWriter;
   }
 
   <#if listName == "LargeList">
@@ -337,18 +336,14 @@ public void writeNull() {
 
   @Override
   public void writeExtension(Object value) {
-    writer.writeExtension(value, extensionType);
-    writer.setPosition(writer.idx() + 1);
+    writer.writeExtension(value);
   }
-  
   @Override
-  public void writeExtension(Object value, ArrowType type) {
-    writeExtension(value);
+  public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) {
+    writer.addExtensionTypeWriterFactory(var1);
   }
-  
   public void write(ExtensionHolder var1) {
     writer.write(var1);
-    writer.setPosition(writer.idx() + 1);
   }
 
   <#list vv.types as type>
