Skip to content

Forward-merge release/26.08 into main #23531

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gforsyth merged 3 commits into from
Aug 4, 2026
Merged

Forward-merge release/26.08 into main #23531

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4a902a2
Upgrade to nvcomp 5.3.0.16 (#23456)
robertmaynard Jul 29, 2026
132eaef
Fix flaky cudf-polars Quent integration test (#23482)
TomAugspurger Jul 31, 2026
c5dc92c
Merge branch 'release/26.08' into main-merge-release/26.08
gforsyth Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-129_arch-aarch64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ dependencies:
- libcufile-dev
- libcurand-dev
- libkvikio==26.10.*,>=0.0.0a0
- libnvcomp-dev==5.2.0.10
- libnvcomp-dev==5.3.0.16
- libnvjitlink-dev
- librapidsmpf==26.10.*,>=0.0.0a0
- librdkafka<2.15.0a0
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-129_arch-x86_64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ dependencies:
- libcufile-dev
- libcurand-dev
- libkvikio==26.10.*,>=0.0.0a0
- libnvcomp-dev==5.2.0.10
- libnvcomp-dev==5.3.0.16
- libnvjitlink-dev
- librapidsmpf==26.10.*,>=0.0.0a0
- librdkafka<2.15.0a0
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-133_arch-aarch64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ dependencies:
- libcufile-dev
- libcurand-dev
- libkvikio==26.10.*,>=0.0.0a0
- libnvcomp-dev==5.2.0.10
- libnvcomp-dev==5.3.0.16
- libnvjitlink-dev
- librapidsmpf==26.10.*,>=0.0.0a0
- librdkafka<2.15.0a0
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-133_arch-x86_64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ dependencies:
- libcufile-dev
- libcurand-dev
- libkvikio==26.10.*,>=0.0.0a0
- libnvcomp-dev==5.2.0.10
- libnvcomp-dev==5.3.0.16
- libnvjitlink-dev
- librapidsmpf==26.10.*,>=0.0.0a0
- librdkafka<2.15.0a0
Expand Down
2 changes: 1 addition & 1 deletion conda/recipes/libcudf/conda_build_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ flatbuffers_version:
- "=24.3.25"

nvcomp_version:
- "=5.2.0.10"
- "=5.3.0.16"

zlib_version:
- ">=1.2.13"
84 changes: 62 additions & 22 deletions cpp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1170,12 +1170,13 @@ set_target_properties(
# and use them.
set_target_properties(
cudf
PROPERTIES EXPORT_PROPERTIES
"CUDF_CXX_FLAGS;CUDF_CUDA_FLAGS;CUDF_CXX_DEFINITIONS;CUDF_CUDA_DEFINITIONS"
CUDF_CXX_FLAGS "${CUDF_CXX_FLAGS}"
CUDF_CUDA_FLAGS "${CUDF_CUDA_FLAGS}"
CUDF_CXX_DEFINITIONS "${CUDF_CXX_DEFINITIONS}"
CUDF_CUDA_DEFINITIONS "${CUDF_CUDA_DEFINITIONS}"
PROPERTIES
EXPORT_PROPERTIES
"CUDF_CXX_FLAGS;CUDF_CUDA_FLAGS;CUDF_CXX_DEFINITIONS;CUDF_CUDA_DEFINITIONS;CUDF_STATIC_LINKED_PROJECTS"
CUDF_CXX_FLAGS "${CUDF_CXX_FLAGS}"
CUDF_CUDA_FLAGS "${CUDF_CUDA_FLAGS}"
CUDF_CXX_DEFINITIONS "${CUDF_CXX_DEFINITIONS}"
CUDF_CUDA_DEFINITIONS "${CUDF_CUDA_DEFINITIONS}"
)

# Note: This must come before the target_compile_options below so that the function can modify the
Expand Down Expand Up @@ -1249,21 +1250,6 @@ target_compile_definitions(cudf PRIVATE $<$<BOOL:${CUDF_KVIKIO_REMOTE_IO}>:CUDF_
# https://github.com/NVIDIA/cccl/pull/2844
target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_AVOID_SORT_UNROLL=1)

# Specify the target module library dependencies
target_link_libraries(
cudf
PUBLIC CCCL::CCCL $<BUILD_LOCAL_INTERFACE:BS::thread_pool>
PRIVATE $<BUILD_LOCAL_INTERFACE:nvtx3::nvtx3-cpp>
$<BUILD_LOCAL_INTERFACE:cuco::cuco>
ZLIB::ZLIB
${CUDF_nvcomp_TARGET}
kvikio::kvikio
${CUDF_nanoarrow_TARGET}
zstd
$<BUILD_LOCAL_INTERFACE:xxhash>
rtcx::rtcx
)

# When rmm is a static library being absorbed via whole-archive, strip nvtx3 from its public
# interface. We bundle nvtx3 headers directly into cudf's install tree, so consumers get them from
# cudf's include path without needing the nvtx3 target or find_dependency(nvtx3).
Expand All @@ -1284,8 +1270,12 @@ endif()
# against them directly. Public absorbed dependencies still promote their public transitive
# dependencies and export set metadata into cudf's interface.
set(_absorbed_deps rmm rapids_logger)
set(_absorbed_nvcomp FALSE)
foreach(_dep IN LISTS _absorbed_deps)
set(_dep_link "${_dep}::${_dep}")
if(NOT TARGET "${_dep}::${_dep}")
continue()
endif()
get_target_property(_dep_link "${_dep}::${_dep}" NAME)
get_target_property(_target_type ${_dep_link} TYPE)
if(NOT BUILD_SHARED_LIBS OR NOT _target_type STREQUAL "STATIC_LIBRARY")
# If not building a shared library, or the dependency is not a static library, link normally.
Expand Down Expand Up @@ -1317,6 +1307,19 @@ foreach(_dep IN LISTS _absorbed_deps)
target_link_libraries(
cudf PRIVATE "$<BUILD_INTERFACE:$<LINK_LIBRARY:WHOLE_ARCHIVE,${_dep_link}>>"
)
set_property(
TARGET cudf
APPEND
PROPERTY CUDF_STATIC_LINKED_PROJECTS "${_dep_link}"
)

# nvcomp_static has done a WHOLE_ARCHIVE on rmm and rapids_logger itself. Therefore it needs to
# come after our WHOLE_ARCHIVE otherwise we will get duplicate symbols for rmm and/or
# rapids_logger
if(TARGET nvcomp::nvcomp_static)
target_link_libraries(${_dep_link} INTERFACE $<BUILD_LOCAL_INTERFACE:nvcomp::nvcomp_static>)
set(_absorbed_nvcomp TRUE)
endif()
Comment on lines +1310 to +1322

@coderabbitai coderabbitai Bot Aug 4, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/usr/bin/env bash
set -euo pipefail

# Trace writes to libcudf static-link metadata.
rg -n -C 6 \
  'CUDF_STATIC_LINKED_PROJECTS|_absorbed_nvcomp|nvcomp::nvcomp_static|CUDF_nvcomp_TARGET' \
  cpp/CMakeLists.txt

# Trace the Java consumer decision.
rg -n -C 6 \
  'cudf_static_linked_projects|nvcomp IN_LIST|nvcomp::nvcomp_static|nvcomp::nvcomp' \
  java/src/main/native/CMakeLists.txt

Repository: rapidsai/cudf

Length of output: 4826

🏁 Script executed:

#!/usr/bin/env bash
set -euo pipefail

printf '%s\n' '--- cpp/CMakeLists.txt relevant sections ---'
sed -n '1240,1400p' cpp/CMakeLists.txt

printf '%s\n' '--- Java native configuration and nvcomp target setup ---'
sed -n '80,125p' java/src/main/native/CMakeLists.txt
sed -n '250,300p' java/src/main/native/CMakeLists.txt

printf '%s\n' '--- all metadata consumers and nvcomp target references ---'
rg -n -C 4 \
  'CUDF_STATIC_LINKED_PROJECTS|CUDF_JNI_LIBCUDF_STATIC|CUDF_nvcomp_TARGET|nvcomp::nvcomp(_static)?' \
  --glob 'CMakeLists.txt' --glob '*.cmake' .

Repository: rapidsai/cudf

Length of output: 22714

🏁 Script executed:

#!/usr/bin/env bash
set -euo pipefail

python3 - <<'PY'
from pathlib import Path

cpp = Path("cpp/CMakeLists.txt").read_text()
java = Path("java/src/main/native/CMakeLists.txt").read_text()

loop = cpp[cpp.index("set(_absorbed_deps"):cpp.index("if(TARGET spdlog::spdlog)")]
assert "set(_absorbed_nvcomp FALSE)" in loop
assert "if(TARGET nvcomp::nvcomp_static)" in loop
assert "set(_absorbed_nvcomp TRUE)" in loop
assert 'PROPERTY CUDF_STATIC_LINKED_PROJECTS "${_dep_link}"' in loop
assert 'PROPERTY CUDF_STATIC_LINKED_PROJECTS "nvcomp"' not in loop

fallback = cpp[cpp.index("if(NOT _absorbed_nvcomp)"):cpp.index("# Add Conda library")]
assert 'PROPERTY CUDF_STATIC_LINKED_PROJECTS "nvcomp"' in fallback

consumer = java[java.index("get_target_property(cudf_static_linked_projects"):
                java.index("target_link_libraries(cudfjni PRIVATE nvtx3::nvtx3-cpp)")]
assert "if(nvcomp IN_LIST cudf_static_linked_projects)" in consumer
assert "nvcomp::nvcomp_static" in consumer
assert "nvcomp::nvcomp" in consumer

# Model the proposed guard across the two absorbed dependencies.
for static_deps in ([], ["rmm"], ["rapids_logger"], ["rmm", "rapids_logger"]):
    absorbed = False
    entries = []
    for dep in static_deps:
        if not absorbed:
            entries.append(dep)
        if True:  # nvcomp::nvcomp_static exists
            if not absorbed:
                entries.append("nvcomp")
            absorbed = True
    assert entries.count("nvcomp") == (1 if static_deps else 0)

print("The current loop omits nvcomp metadata on the absorption path.")
print("The proposed guard records nvcomp once for any absorbed static dependency.")
print("The Java consumer uses nvcomp::nvcomp_static only when nvcomp is in the exported list.")
PY

Repository: rapidsai/cudf

Length of output: 376

Export nvcomp for the absorption path.

When nvcomp::nvcomp_static exists, this branch sets _absorbed_nvcomp, so the later block does not export nvcomp. The Java build then selects nvcomp::nvcomp; if only the static target exists, CMake cannot resolve that target. Append nvcomp once before setting _absorbed_nvcomp to TRUE.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@cpp/CMakeLists.txt` around lines 1310 - 1322, Update the nvcomp absorption
branch around target_link_libraries and _absorbed_nvcomp so it appends the
nvcomp dependency to the export list before setting _absorbed_nvcomp to TRUE.
Ensure nvcomp is exported exactly once when only nvcomp::nvcomp_static exists,
allowing the Java build’s nvcomp::nvcomp reference to resolve.


# Merge the absorbed library's export set metadata into cudf-exports so the installed config has
# find_dependency() calls for all transitive deps (e.g. nvtx3, CUDAToolkit from rmm).
Expand Down Expand Up @@ -1346,6 +1349,43 @@ foreach(_dep IN LISTS _absorbed_deps)
endforeach()
endforeach()

if(TARGET spdlog::spdlog)
get_target_property(_spdlog_link spdlog::spdlog NAME)
get_target_property(_spdlog_type ${_spdlog_link} TYPE)
if(BUILD_SHARED_LIBS AND _spdlog_type STREQUAL "STATIC_LIBRARY")
target_link_libraries(
cudf PRIVATE "$<BUILD_INTERFACE:$<LINK_LIBRARY:WHOLE_ARCHIVE,${_spdlog_link}>>"
)
set_property(
TARGET cudf
APPEND
PROPERTY CUDF_STATIC_LINKED_PROJECTS "${_spdlog_link}"
)
else()
target_link_libraries(cudf PRIVATE ${_spdlog_link})
endif()
endif()

# Specify the target module library dependencies We do this after all the WHOLE_ARCHIVES to protect
# against any of these libraries ever adding a rapids_logger / rmm hidden dependency
target_link_libraries(
cudf
PUBLIC CCCL::CCCL $<BUILD_LOCAL_INTERFACE:BS::thread_pool>
PRIVATE $<BUILD_LOCAL_INTERFACE:nvtx3::nvtx3-cpp> $<BUILD_LOCAL_INTERFACE:cuco::cuco> ZLIB::ZLIB
kvikio::kvikio ${CUDF_nanoarrow_TARGET} zstd $<BUILD_LOCAL_INTERFACE:xxhash> rtcx::rtcx
)

if(NOT _absorbed_nvcomp)
target_link_libraries(cudf PRIVATE ${CUDF_nvcomp_TARGET})
if(CUDF_nvcomp_TARGET STREQUAL "nvcomp::nvcomp_static")
set_property(
TARGET cudf
APPEND
PROPERTY CUDF_STATIC_LINKED_PROJECTS "nvcomp"
)
endif()
endif()

# Add Conda library, and include paths if specified
if(TARGET conda_env)
target_link_libraries(cudf PRIVATE conda_env)
Expand Down
4 changes: 2 additions & 2 deletions cpp/cmake/thirdparty/get_nvcomp.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# =============================================================================
# cmake-format: off
# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION.
# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# cmake-format: on
# =============================================================================
Expand Down Expand Up @@ -127,7 +127,7 @@ function(find_and_configure_nvcomp)
endif()
endfunction()

set(_nvcomp_args VERSION 5.2.0.10)
set(_nvcomp_args VERSION 5.3.0.16)
if(CUDF_BUILD_STATIC_DEPS STREQUAL "FORCE")
list(APPEND _nvcomp_args DOWNLOAD_ONLY)
endif()
Expand Down
8 changes: 4 additions & 4 deletions dependencies.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -778,7 +778,7 @@ dependencies:
- output_types: conda
packages:
# Align nvcomp version with rapids-cmake
- libnvcomp-dev==5.2.0.10
- libnvcomp-dev==5.3.0.16
specific:
- output_types: [requirements, pyproject]
matrices:
Expand All @@ -789,16 +789,16 @@ dependencies:
cuda: "12.*"
use_cuda_wheels: "true"
packages:
- nvidia-libnvcomp-cu12==5.2.0.13
- nvidia-libnvcomp-cu12==5.3.0.16
- matrix:
cuda: "13.*"
use_cuda_wheels: "true"
packages:
- nvidia-libnvcomp-cu13==5.2.0.13
- nvidia-libnvcomp-cu13==5.3.0.16
- matrix:
use_cuda_wheels: "true"
packages:
- nvidia-libnvcomp==5.2.0.13
- nvidia-libnvcomp==5.3.0.16
rapids_build_skbuild:
common:
- output_types: [conda, requirements, pyproject]
Expand Down
40 changes: 18 additions & 22 deletions java/src/main/native/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,11 @@ endif()
find_package(nvtx3 REQUIRED)

if(NOT TARGET nvcomp
AND NOT TARGET nvcomp::nvcomp
AND NOT TARGET nvcomp::nvcomp_static
AND NOT DEFINED ENV{CUDF_INSTALL_DIR}
)
include(${CUDF_SOURCE_DIR}/cmake/thirdparty/get_nvcomp.cmake)
endif()
find_package(nvcomp REQUIRED)

# ##################################################################################################
# * find JNI --------------------------------------------------------------------------------------
Expand Down Expand Up @@ -268,30 +267,19 @@ endif()

# ##################################################################################################
# * link libraries --------------------------------------------------------------------------------

set(CUDF_LINK PUBLIC cudf::cudf)
if(CUDF_JNI_LIBCUDF_STATIC)
# Whole-link libcudf.a into the shared library but not its dependencies
set(CUDF_LINK PRIVATE -Wl,--whole-archive cudf::cudf -Wl,--no-whole-archive)
target_link_libraries(cudfjni PRIVATE $<LINK_LIBRARY:WHOLE_ARCHIVE,cudf::cudf>)
else()
target_link_libraries(cudfjni PUBLIC cudf::cudf)
endif()

# When nvcomp is installed we need to use nvcomp::nvcomp but from the cudf build directory it will
# just be nvcomp.
target_link_libraries(
cudfjni ${CUDF_LINK} PRIVATE nvtx3::nvtx3-cpp $<TARGET_NAME_IF_EXISTS:nvcomp>
$<TARGET_NAME_IF_EXISTS:nvcomp::nvcomp>
)

# ##################################################################################################
# * cudart options --------------------------------------------------------------------------------
# cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic
# linking

set_target_properties(cudfjni PROPERTIES CUDA_RUNTIME_LIBRARY Static)
get_target_property(cudf_static_linked_projects cudf::cudf CUDF_STATIC_LINKED_PROJECTS)
if(nvcomp IN_LIST cudf_static_linked_projects)
target_link_libraries(cudfjni PRIVATE $<COMPILE_ONLY:nvcomp::nvcomp_static>)
else()
target_link_libraries(cudfjni PRIVATE nvcomp::nvcomp)

# ##################################################################################################
# * install shared libraries ----------------------------------------------------------------------
if(TARGET nvcomp::nvcomp)
# * install nvcomp for tests
add_custom_command(
TARGET cudfjni
PRE_LINK
Expand All @@ -300,3 +288,11 @@ if(TARGET nvcomp::nvcomp)
COMMENT "Copying nvcomp libraries to ${PROJECT_BINARY_DIR}"
)
endif()

target_link_libraries(cudfjni PRIVATE nvtx3::nvtx3-cpp)

# ##################################################################################################
# * cudart options --------------------------------------------------------------------------------
# cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic
# linking
set_target_properties(cudfjni PROPERTIES CUDA_RUNTIME_LIBRARY Static)
12 changes: 10 additions & 2 deletions python/cudf_polars/tests/quent/test_quent_integration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from __future__ import annotations

from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any

import pytest

Expand All @@ -26,6 +26,8 @@
def engine_with_quent_context(
request: pytest.FixtureRequest,
quent_context: QuentContext,
ray_num_ranks: int,
ray_init_options: dict[str, Any],
) -> Iterator[StreamingEngine]:
"""
A streaming engine configured with a quent context from the 'quent_context'
Expand All @@ -43,8 +45,14 @@ def engine_with_quent_context(
pytest.importorskip("ray")
import cudf_polars.engine.ray

# Always specify num_ranks: the default path sizes the engine from the
# GPUs Ray reports, which fails if this test shares an xdist worker
# with a test that already brought up the shared num_gpus=0 cluster.
engine = cudf_polars.engine.ray.RayEngine(
executor_options={"quent_context": quent_context}
executor_options={"quent_context": quent_context},
engine_options={"allow_gpu_sharing": True},
ray_init_options=ray_init_options,
num_ranks=ray_num_ranks,
)
elif backend == "dask":
pytest.importorskip("distributed")
Expand Down
4 changes: 2 additions & 2 deletions python/libcudf/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ classifiers = [
dependencies = [
"libkvikio==26.10.*,>=0.0.0a0",
"librmm==26.10.*,>=0.0.0a0",
"nvidia-libnvcomp==5.2.0.13",
"nvidia-libnvcomp==5.3.0.16",
"nvidia-nvjitlink>=13.3,<14",
"rapids-logger==0.2.*,>=0.0.0a0",
] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
Expand Down Expand Up @@ -73,7 +73,7 @@ requires = [
"libkvikio==26.10.*,>=0.0.0a0",
"librmm==26.10.*,>=0.0.0a0",
"ninja",
"nvidia-libnvcomp==5.2.0.13",
"nvidia-libnvcomp==5.3.0.16",
"nvidia-nvjitlink>=13.3,<14",
"rapids-logger==0.2.*,>=0.0.0a0",
] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
Loading