Forward-merge release/26.08 into main

conda/environments/all_cuda-129_arch-aarch64.yaml

@@ -45,7 +45,7 @@ dependencies:
 - libcufile-dev
 - libcurand-dev
 - libkvikio==26.8.*,>=0.0.0a0
-- libnvcomp-dev==5.2.0.10
+- libnvcomp-dev==5.3.0.16
 - libnvjitlink-dev
 - librapidsmpf==26.8.*,>=0.0.0a0
 - librdkafka<2.15.0a0

conda/environments/all_cuda-129_arch-x86_64.yaml

@@ -45,7 +45,7 @@ dependencies:
 - libcufile-dev
 - libcurand-dev
 - libkvikio==26.8.*,>=0.0.0a0
-- libnvcomp-dev==5.2.0.10
+- libnvcomp-dev==5.3.0.16
 - libnvjitlink-dev
 - librapidsmpf==26.8.*,>=0.0.0a0
 - librdkafka<2.15.0a0

conda/environments/all_cuda-133_arch-aarch64.yaml

@@ -45,7 +45,7 @@ dependencies:
 - libcufile-dev
 - libcurand-dev
 - libkvikio==26.8.*,>=0.0.0a0
-- libnvcomp-dev==5.2.0.10
+- libnvcomp-dev==5.3.0.16
 - libnvjitlink-dev
 - librapidsmpf==26.8.*,>=0.0.0a0
 - librdkafka<2.15.0a0

conda/environments/all_cuda-133_arch-x86_64.yaml

@@ -45,7 +45,7 @@ dependencies:
 - libcufile-dev
 - libcurand-dev
 - libkvikio==26.8.*,>=0.0.0a0
-- libnvcomp-dev==5.2.0.10
+- libnvcomp-dev==5.3.0.16
 - libnvjitlink-dev
 - librapidsmpf==26.8.*,>=0.0.0a0
 - librdkafka<2.15.0a0

conda/recipes/libcudf/conda_build_config.yaml

@@ -26,7 +26,7 @@ flatbuffers_version:
   - "=24.3.25"
 
 nvcomp_version:
-  - "=5.2.0.10"
+  - "=5.3.0.16"
 
 zlib_version:
   - ">=1.2.13"

cpp/CMakeLists.txt

@@ -1165,12 +1165,13 @@ set_target_properties(
 # and use them.
 set_target_properties(
   cudf
-  PROPERTIES EXPORT_PROPERTIES
-             "CUDF_CXX_FLAGS;CUDF_CUDA_FLAGS;CUDF_CXX_DEFINITIONS;CUDF_CUDA_DEFINITIONS"
-             CUDF_CXX_FLAGS "${CUDF_CXX_FLAGS}"
-             CUDF_CUDA_FLAGS "${CUDF_CUDA_FLAGS}"
-             CUDF_CXX_DEFINITIONS "${CUDF_CXX_DEFINITIONS}"
-             CUDF_CUDA_DEFINITIONS "${CUDF_CUDA_DEFINITIONS}"
+  PROPERTIES
+    EXPORT_PROPERTIES
+    "CUDF_CXX_FLAGS;CUDF_CUDA_FLAGS;CUDF_CXX_DEFINITIONS;CUDF_CUDA_DEFINITIONS;CUDF_STATIC_LINKED_PROJECTS"
+    CUDF_CXX_FLAGS "${CUDF_CXX_FLAGS}"
+    CUDF_CUDA_FLAGS "${CUDF_CUDA_FLAGS}"
+    CUDF_CXX_DEFINITIONS "${CUDF_CXX_DEFINITIONS}"
+    CUDF_CUDA_DEFINITIONS "${CUDF_CUDA_DEFINITIONS}"
 )
 
 # Note: This must come before the target_compile_options below so that the function can modify the
@@ -1244,21 +1245,6 @@ target_compile_definitions(cudf PRIVATE $<$<BOOL:${CUDF_KVIKIO_REMOTE_IO}>:CUDF_
 # https://github.com/NVIDIA/cccl/pull/2844
 target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_AVOID_SORT_UNROLL=1)
 
-# Specify the target module library dependencies
-target_link_libraries(
-  cudf
-  PUBLIC CCCL::CCCL $<BUILD_LOCAL_INTERFACE:BS::thread_pool>
-  PRIVATE $<BUILD_LOCAL_INTERFACE:nvtx3::nvtx3-cpp>
-          $<BUILD_LOCAL_INTERFACE:cuco::cuco>
-          ZLIB::ZLIB
-          ${CUDF_nvcomp_TARGET}
-          kvikio::kvikio
-          ${CUDF_nanoarrow_TARGET}
-          zstd
-          $<BUILD_LOCAL_INTERFACE:xxhash>
-          rtcx::rtcx
-)
-
 # When rmm is a static library being absorbed via whole-archive, strip nvtx3 from its public
 # interface. We bundle nvtx3 headers directly into cudf's install tree, so consumers get them from
 # cudf's include path without needing the nvtx3 target or find_dependency(nvtx3).
@@ -1279,8 +1265,12 @@ endif()
 # against them directly. Public absorbed dependencies still promote their public transitive
 # dependencies and export set metadata into cudf's interface.
 set(_absorbed_deps rmm rapids_logger)
+set(_absorbed_nvcomp FALSE)
 foreach(_dep IN LISTS _absorbed_deps)
-  set(_dep_link "${_dep}::${_dep}")
+  if(NOT TARGET "${_dep}::${_dep}")
+    continue()
+  endif()
+  get_target_property(_dep_link "${_dep}::${_dep}" NAME)
   get_target_property(_target_type ${_dep_link} TYPE)
   if(NOT BUILD_SHARED_LIBS OR NOT _target_type STREQUAL "STATIC_LIBRARY")
     # If not building a shared library, or the dependency is not a static library, link normally.
@@ -1312,6 +1302,19 @@ foreach(_dep IN LISTS _absorbed_deps)
   target_link_libraries(
     cudf PRIVATE "$<BUILD_INTERFACE:$<LINK_LIBRARY:WHOLE_ARCHIVE,${_dep_link}>>"
   )
+  set_property(
+    TARGET cudf
+    APPEND
+    PROPERTY CUDF_STATIC_LINKED_PROJECTS "${_dep_link}"
+  )
+
+  # nvcomp_static has done a WHOLE_ARCHIVE on rmm and rapids_logger itself. Therefore it needs to
+  # come after our WHOLE_ARCHIVE otherwise we will get duplicate symbols for rmm and/or
+  # rapids_logger
+  if(TARGET nvcomp::nvcomp_static)
+    target_link_libraries(${_dep_link} INTERFACE $<BUILD_LOCAL_INTERFACE:nvcomp::nvcomp_static>)
+    set(_absorbed_nvcomp TRUE)
+  endif()
 
   # Merge the absorbed library's export set metadata into cudf-exports so the installed config has
   # find_dependency() calls for all transitive deps (e.g. nvtx3, CUDAToolkit from rmm).
@@ -1341,6 +1344,43 @@ foreach(_dep IN LISTS _absorbed_deps)
   endforeach()
 endforeach()
 
+if(TARGET spdlog::spdlog)
+  get_target_property(_spdlog_link spdlog::spdlog NAME)
+  get_target_property(_spdlog_type ${_spdlog_link} TYPE)
+  if(BUILD_SHARED_LIBS AND _spdlog_type STREQUAL "STATIC_LIBRARY")
+    target_link_libraries(
+      cudf PRIVATE "$<BUILD_INTERFACE:$<LINK_LIBRARY:WHOLE_ARCHIVE,${_spdlog_link}>>"
+    )
+    set_property(
+      TARGET cudf
+      APPEND
+      PROPERTY CUDF_STATIC_LINKED_PROJECTS "${_spdlog_link}"
+    )
+  else()
+    target_link_libraries(cudf PRIVATE ${_spdlog_link})
+  endif()
+endif()
+
+# Specify the target module library dependencies We do this after all the WHOLE_ARCHIVES to protect
+# against any of these libraries ever adding a rapids_logger / rmm hidden dependency
+target_link_libraries(
+  cudf
+  PUBLIC CCCL::CCCL $<BUILD_LOCAL_INTERFACE:BS::thread_pool>
+  PRIVATE $<BUILD_LOCAL_INTERFACE:nvtx3::nvtx3-cpp> $<BUILD_LOCAL_INTERFACE:cuco::cuco> ZLIB::ZLIB
+          kvikio::kvikio ${CUDF_nanoarrow_TARGET} zstd $<BUILD_LOCAL_INTERFACE:xxhash> rtcx::rtcx
+)
+
+if(NOT _absorbed_nvcomp)
+  target_link_libraries(cudf PRIVATE ${CUDF_nvcomp_TARGET})
+  if(CUDF_nvcomp_TARGET STREQUAL "nvcomp::nvcomp_static")
+    set_property(
+      TARGET cudf
+      APPEND
+      PROPERTY CUDF_STATIC_LINKED_PROJECTS "nvcomp"
+    )
+  endif()
+endif()
+
 # Add Conda library, and include paths if specified
 if(TARGET conda_env)
   target_link_libraries(cudf PRIVATE conda_env)

cpp/cmake/thirdparty/get_nvcomp.cmake

@@ -1,6 +1,6 @@
 # =============================================================================
 # cmake-format: off
-# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION.
+# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 # cmake-format: on
 # =============================================================================
@@ -127,7 +127,7 @@ function(find_and_configure_nvcomp)
   endif()
 endfunction()
 
-set(_nvcomp_args VERSION 5.2.0.10)
+set(_nvcomp_args VERSION 5.3.0.16)
 if(CUDF_BUILD_STATIC_DEPS STREQUAL "FORCE")
   list(APPEND _nvcomp_args DOWNLOAD_ONLY)
 endif()

dependencies.yaml

@@ -661,7 +661,7 @@ dependencies:
       - output_types: conda
         packages:
           # Align nvcomp version with rapids-cmake
-          - libnvcomp-dev==5.2.0.10
+          - libnvcomp-dev==5.3.0.16
     specific:
       - output_types: [requirements, pyproject]
         matrices:
@@ -671,14 +671,14 @@ dependencies:
           - matrix:
               cuda: "12.*"
             packages:
-              - nvidia-libnvcomp-cu12==5.2.0.13
+              - nvidia-libnvcomp-cu12==5.3.0.16
           - matrix:
               cuda: "13.*"
             packages:
-              - nvidia-libnvcomp-cu13==5.2.0.13
+              - nvidia-libnvcomp-cu13==5.3.0.16
           - matrix:
             packages:
-              - nvidia-libnvcomp==5.2.0.13
+              - nvidia-libnvcomp==5.3.0.16
   rapids_build_skbuild:
     common:
       - output_types: [conda, requirements, pyproject]

java/src/main/native/CMakeLists.txt

@@ -109,12 +109,11 @@ endif()
 find_package(nvtx3 REQUIRED)
 
 if(NOT TARGET nvcomp
-   AND NOT TARGET nvcomp::nvcomp
+   AND NOT TARGET nvcomp::nvcomp_static
    AND NOT DEFINED ENV{CUDF_INSTALL_DIR}
 )
   include(${CUDF_SOURCE_DIR}/cmake/thirdparty/get_nvcomp.cmake)
 endif()
-find_package(nvcomp REQUIRED)
 
 # ##################################################################################################
 # * find JNI --------------------------------------------------------------------------------------
@@ -268,30 +267,19 @@ endif()
 
 # ##################################################################################################
 # * link libraries --------------------------------------------------------------------------------
-
-set(CUDF_LINK PUBLIC cudf::cudf)
 if(CUDF_JNI_LIBCUDF_STATIC)
-  # Whole-link libcudf.a into the shared library but not its dependencies
-  set(CUDF_LINK PRIVATE -Wl,--whole-archive cudf::cudf -Wl,--no-whole-archive)
+  target_link_libraries(cudfjni PRIVATE $<LINK_LIBRARY:WHOLE_ARCHIVE,cudf::cudf>)
+else()
+  target_link_libraries(cudfjni PUBLIC cudf::cudf)
 endif()
 
-# When nvcomp is installed we need to use nvcomp::nvcomp but from the cudf build directory it will
-# just be nvcomp.
-target_link_libraries(
-  cudfjni ${CUDF_LINK} PRIVATE nvtx3::nvtx3-cpp $<TARGET_NAME_IF_EXISTS:nvcomp>
-                               $<TARGET_NAME_IF_EXISTS:nvcomp::nvcomp>
-)
-
-# ##################################################################################################
-# * cudart options --------------------------------------------------------------------------------
-#   cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic
-#   linking
-
-set_target_properties(cudfjni PROPERTIES CUDA_RUNTIME_LIBRARY Static)
+get_target_property(cudf_static_linked_projects cudf::cudf CUDF_STATIC_LINKED_PROJECTS)
+if(nvcomp IN_LIST cudf_static_linked_projects)
+  target_link_libraries(cudfjni PRIVATE $<COMPILE_ONLY:nvcomp::nvcomp_static>)
+else()
+  target_link_libraries(cudfjni PRIVATE nvcomp::nvcomp)
 
-# ##################################################################################################
-# * install shared libraries ----------------------------------------------------------------------
-if(TARGET nvcomp::nvcomp)
+  # * install nvcomp for tests
   add_custom_command(
     TARGET cudfjni
     PRE_LINK
@@ -300,3 +288,11 @@ if(TARGET nvcomp::nvcomp)
     COMMENT "Copying nvcomp libraries to ${PROJECT_BINARY_DIR}"
   )
 endif()
+
+target_link_libraries(cudfjni PRIVATE nvtx3::nvtx3-cpp)
+
+# ##################################################################################################
+# * cudart options --------------------------------------------------------------------------------
+#   cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic
+#   linking
+set_target_properties(cudfjni PROPERTIES CUDA_RUNTIME_LIBRARY Static)

python/cudf_polars/tests/quent/test_quent_integration.py

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import pytest
 
@@ -26,6 +26,8 @@
 def engine_with_quent_context(
     request: pytest.FixtureRequest,
     quent_context: QuentContext,
+    ray_num_ranks: int,
+    ray_init_options: dict[str, Any],
 ) -> Iterator[StreamingEngine]:
     """
     A streaming engine configured with a quent context from the 'quent_context'
@@ -43,8 +45,14 @@ def engine_with_quent_context(
         pytest.importorskip("ray")
         import cudf_polars.engine.ray
 
+        # Always specify num_ranks: the default path sizes the engine from the
+        # GPUs Ray reports, which fails if this test shares an xdist worker
+        # with a test that already brought up the shared num_gpus=0 cluster.
         engine = cudf_polars.engine.ray.RayEngine(
-            executor_options={"quent_context": quent_context}
+            executor_options={"quent_context": quent_context},
+            engine_options={"allow_gpu_sharing": True},
+            ray_init_options=ray_init_options,
+            num_ranks=ray_num_ranks,
         )
     elif backend == "dask":
         pytest.importorskip("distributed")

python/libcudf/pyproject.toml

@@ -28,7 +28,7 @@ classifiers = [
 dependencies = [
     "libkvikio==26.8.*,>=0.0.0a0",
     "librmm==26.8.*,>=0.0.0a0",
-    "nvidia-libnvcomp==5.2.0.13",
+    "nvidia-libnvcomp==5.3.0.16",
     "nvidia-nvjitlink>=13.3,<14",
     "rapids-logger==0.2.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
@@ -73,7 +73,7 @@ requires = [
     "libkvikio==26.8.*,>=0.0.0a0",
     "librmm==26.8.*,>=0.0.0a0",
     "ninja",
-    "nvidia-libnvcomp==5.2.0.13",
+    "nvidia-libnvcomp==5.3.0.16",
     "nvidia-nvjitlink>=13.3,<14",
     "rapids-logger==0.2.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.