diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 8bd33e02708a..02b4ef9ff197 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -46,7 +46,7 @@ dependencies: - libcufile-dev - libcurand-dev - libkvikio==26.10.*,>=0.0.0a0 -- libnvcomp-dev==5.2.0.10 +- libnvcomp-dev==5.3.0.16 - libnvjitlink-dev - librapidsmpf==26.10.*,>=0.0.0a0 - librdkafka<2.15.0a0 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 036e454bd1d4..9e1de949a4d9 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -46,7 +46,7 @@ dependencies: - libcufile-dev - libcurand-dev - libkvikio==26.10.*,>=0.0.0a0 -- libnvcomp-dev==5.2.0.10 +- libnvcomp-dev==5.3.0.16 - libnvjitlink-dev - librapidsmpf==26.10.*,>=0.0.0a0 - librdkafka<2.15.0a0 diff --git a/conda/environments/all_cuda-133_arch-aarch64.yaml b/conda/environments/all_cuda-133_arch-aarch64.yaml index c4a82ebaa4bf..aa7fda73c554 100644 --- a/conda/environments/all_cuda-133_arch-aarch64.yaml +++ b/conda/environments/all_cuda-133_arch-aarch64.yaml @@ -46,7 +46,7 @@ dependencies: - libcufile-dev - libcurand-dev - libkvikio==26.10.*,>=0.0.0a0 -- libnvcomp-dev==5.2.0.10 +- libnvcomp-dev==5.3.0.16 - libnvjitlink-dev - librapidsmpf==26.10.*,>=0.0.0a0 - librdkafka<2.15.0a0 diff --git a/conda/environments/all_cuda-133_arch-x86_64.yaml b/conda/environments/all_cuda-133_arch-x86_64.yaml index 393f7040dd9e..b4eb46fd449c 100644 --- a/conda/environments/all_cuda-133_arch-x86_64.yaml +++ b/conda/environments/all_cuda-133_arch-x86_64.yaml @@ -46,7 +46,7 @@ dependencies: - libcufile-dev - libcurand-dev - libkvikio==26.10.*,>=0.0.0a0 -- libnvcomp-dev==5.2.0.10 +- libnvcomp-dev==5.3.0.16 - libnvjitlink-dev - librapidsmpf==26.10.*,>=0.0.0a0 - librdkafka<2.15.0a0 diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 390f22cc2de0..4f758589e239 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -26,7 +26,7 @@ flatbuffers_version: - "=24.3.25" nvcomp_version: - - "=5.2.0.10" + - "=5.3.0.16" zlib_version: - ">=1.2.13" diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9edbb51e8820..862c7f986290 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1170,12 +1170,13 @@ set_target_properties( # and use them. set_target_properties( cudf - PROPERTIES EXPORT_PROPERTIES - "CUDF_CXX_FLAGS;CUDF_CUDA_FLAGS;CUDF_CXX_DEFINITIONS;CUDF_CUDA_DEFINITIONS" - CUDF_CXX_FLAGS "${CUDF_CXX_FLAGS}" - CUDF_CUDA_FLAGS "${CUDF_CUDA_FLAGS}" - CUDF_CXX_DEFINITIONS "${CUDF_CXX_DEFINITIONS}" - CUDF_CUDA_DEFINITIONS "${CUDF_CUDA_DEFINITIONS}" + PROPERTIES + EXPORT_PROPERTIES + "CUDF_CXX_FLAGS;CUDF_CUDA_FLAGS;CUDF_CXX_DEFINITIONS;CUDF_CUDA_DEFINITIONS;CUDF_STATIC_LINKED_PROJECTS" + CUDF_CXX_FLAGS "${CUDF_CXX_FLAGS}" + CUDF_CUDA_FLAGS "${CUDF_CUDA_FLAGS}" + CUDF_CXX_DEFINITIONS "${CUDF_CXX_DEFINITIONS}" + CUDF_CUDA_DEFINITIONS "${CUDF_CUDA_DEFINITIONS}" ) # Note: This must come before the target_compile_options below so that the function can modify the @@ -1249,21 +1250,6 @@ target_compile_definitions(cudf PRIVATE $<$:CUDF_ # https://github.com/NVIDIA/cccl/pull/2844 target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_AVOID_SORT_UNROLL=1) -# Specify the target module library dependencies -target_link_libraries( - cudf - PUBLIC CCCL::CCCL $ - PRIVATE $ - $ - ZLIB::ZLIB - ${CUDF_nvcomp_TARGET} - kvikio::kvikio - ${CUDF_nanoarrow_TARGET} - zstd - $ - rtcx::rtcx -) - # When rmm is a static library being absorbed via whole-archive, strip nvtx3 from its public # interface. We bundle nvtx3 headers directly into cudf's install tree, so consumers get them from # cudf's include path without needing the nvtx3 target or find_dependency(nvtx3). @@ -1284,8 +1270,12 @@ endif() # against them directly. Public absorbed dependencies still promote their public transitive # dependencies and export set metadata into cudf's interface. set(_absorbed_deps rmm rapids_logger) +set(_absorbed_nvcomp FALSE) foreach(_dep IN LISTS _absorbed_deps) - set(_dep_link "${_dep}::${_dep}") + if(NOT TARGET "${_dep}::${_dep}") + continue() + endif() + get_target_property(_dep_link "${_dep}::${_dep}" NAME) get_target_property(_target_type ${_dep_link} TYPE) if(NOT BUILD_SHARED_LIBS OR NOT _target_type STREQUAL "STATIC_LIBRARY") # If not building a shared library, or the dependency is not a static library, link normally. @@ -1317,6 +1307,19 @@ foreach(_dep IN LISTS _absorbed_deps) target_link_libraries( cudf PRIVATE "$>" ) + set_property( + TARGET cudf + APPEND + PROPERTY CUDF_STATIC_LINKED_PROJECTS "${_dep_link}" + ) + + # nvcomp_static has done a WHOLE_ARCHIVE on rmm and rapids_logger itself. Therefore it needs to + # come after our WHOLE_ARCHIVE otherwise we will get duplicate symbols for rmm and/or + # rapids_logger + if(TARGET nvcomp::nvcomp_static) + target_link_libraries(${_dep_link} INTERFACE $) + set(_absorbed_nvcomp TRUE) + endif() # Merge the absorbed library's export set metadata into cudf-exports so the installed config has # find_dependency() calls for all transitive deps (e.g. nvtx3, CUDAToolkit from rmm). @@ -1346,6 +1349,43 @@ foreach(_dep IN LISTS _absorbed_deps) endforeach() endforeach() +if(TARGET spdlog::spdlog) + get_target_property(_spdlog_link spdlog::spdlog NAME) + get_target_property(_spdlog_type ${_spdlog_link} TYPE) + if(BUILD_SHARED_LIBS AND _spdlog_type STREQUAL "STATIC_LIBRARY") + target_link_libraries( + cudf PRIVATE "$>" + ) + set_property( + TARGET cudf + APPEND + PROPERTY CUDF_STATIC_LINKED_PROJECTS "${_spdlog_link}" + ) + else() + target_link_libraries(cudf PRIVATE ${_spdlog_link}) + endif() +endif() + +# Specify the target module library dependencies We do this after all the WHOLE_ARCHIVES to protect +# against any of these libraries ever adding a rapids_logger / rmm hidden dependency +target_link_libraries( + cudf + PUBLIC CCCL::CCCL $ + PRIVATE $ $ ZLIB::ZLIB + kvikio::kvikio ${CUDF_nanoarrow_TARGET} zstd $ rtcx::rtcx +) + +if(NOT _absorbed_nvcomp) + target_link_libraries(cudf PRIVATE ${CUDF_nvcomp_TARGET}) + if(CUDF_nvcomp_TARGET STREQUAL "nvcomp::nvcomp_static") + set_property( + TARGET cudf + APPEND + PROPERTY CUDF_STATIC_LINKED_PROJECTS "nvcomp" + ) + endif() +endif() + # Add Conda library, and include paths if specified if(TARGET conda_env) target_link_libraries(cudf PRIVATE conda_env) diff --git a/cpp/cmake/thirdparty/get_nvcomp.cmake b/cpp/cmake/thirdparty/get_nvcomp.cmake index 248595ca17cc..522a2050256e 100644 --- a/cpp/cmake/thirdparty/get_nvcomp.cmake +++ b/cpp/cmake/thirdparty/get_nvcomp.cmake @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -127,7 +127,7 @@ function(find_and_configure_nvcomp) endif() endfunction() -set(_nvcomp_args VERSION 5.2.0.10) +set(_nvcomp_args VERSION 5.3.0.16) if(CUDF_BUILD_STATIC_DEPS STREQUAL "FORCE") list(APPEND _nvcomp_args DOWNLOAD_ONLY) endif() diff --git a/dependencies.yaml b/dependencies.yaml index 78aae5419453..d14a559a977c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -778,7 +778,7 @@ dependencies: - output_types: conda packages: # Align nvcomp version with rapids-cmake - - libnvcomp-dev==5.2.0.10 + - libnvcomp-dev==5.3.0.16 specific: - output_types: [requirements, pyproject] matrices: @@ -789,16 +789,16 @@ dependencies: cuda: "12.*" use_cuda_wheels: "true" packages: - - nvidia-libnvcomp-cu12==5.2.0.13 + - nvidia-libnvcomp-cu12==5.3.0.16 - matrix: cuda: "13.*" use_cuda_wheels: "true" packages: - - nvidia-libnvcomp-cu13==5.2.0.13 + - nvidia-libnvcomp-cu13==5.3.0.16 - matrix: use_cuda_wheels: "true" packages: - - nvidia-libnvcomp==5.2.0.13 + - nvidia-libnvcomp==5.3.0.16 rapids_build_skbuild: common: - output_types: [conda, requirements, pyproject] diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index c0234bf6d620..ed4b17835346 100644 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -109,12 +109,11 @@ endif() find_package(nvtx3 REQUIRED) if(NOT TARGET nvcomp - AND NOT TARGET nvcomp::nvcomp + AND NOT TARGET nvcomp::nvcomp_static AND NOT DEFINED ENV{CUDF_INSTALL_DIR} ) include(${CUDF_SOURCE_DIR}/cmake/thirdparty/get_nvcomp.cmake) endif() -find_package(nvcomp REQUIRED) # ################################################################################################## # * find JNI -------------------------------------------------------------------------------------- @@ -268,30 +267,19 @@ endif() # ################################################################################################## # * link libraries -------------------------------------------------------------------------------- - -set(CUDF_LINK PUBLIC cudf::cudf) if(CUDF_JNI_LIBCUDF_STATIC) - # Whole-link libcudf.a into the shared library but not its dependencies - set(CUDF_LINK PRIVATE -Wl,--whole-archive cudf::cudf -Wl,--no-whole-archive) + target_link_libraries(cudfjni PRIVATE $) +else() + target_link_libraries(cudfjni PUBLIC cudf::cudf) endif() -# When nvcomp is installed we need to use nvcomp::nvcomp but from the cudf build directory it will -# just be nvcomp. -target_link_libraries( - cudfjni ${CUDF_LINK} PRIVATE nvtx3::nvtx3-cpp $ - $ -) - -# ################################################################################################## -# * cudart options -------------------------------------------------------------------------------- -# cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic -# linking - -set_target_properties(cudfjni PROPERTIES CUDA_RUNTIME_LIBRARY Static) +get_target_property(cudf_static_linked_projects cudf::cudf CUDF_STATIC_LINKED_PROJECTS) +if(nvcomp IN_LIST cudf_static_linked_projects) + target_link_libraries(cudfjni PRIVATE $) +else() + target_link_libraries(cudfjni PRIVATE nvcomp::nvcomp) -# ################################################################################################## -# * install shared libraries ---------------------------------------------------------------------- -if(TARGET nvcomp::nvcomp) + # * install nvcomp for tests add_custom_command( TARGET cudfjni PRE_LINK @@ -300,3 +288,11 @@ if(TARGET nvcomp::nvcomp) COMMENT "Copying nvcomp libraries to ${PROJECT_BINARY_DIR}" ) endif() + +target_link_libraries(cudfjni PRIVATE nvtx3::nvtx3-cpp) + +# ################################################################################################## +# * cudart options -------------------------------------------------------------------------------- +# cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic +# linking +set_target_properties(cudfjni PROPERTIES CUDA_RUNTIME_LIBRARY Static) diff --git a/python/cudf_polars/tests/quent/test_quent_integration.py b/python/cudf_polars/tests/quent/test_quent_integration.py index d6fb79b30185..3c4c53710e93 100644 --- a/python/cudf_polars/tests/quent/test_quent_integration.py +++ b/python/cudf_polars/tests/quent/test_quent_integration.py @@ -5,7 +5,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import pytest @@ -26,6 +26,8 @@ def engine_with_quent_context( request: pytest.FixtureRequest, quent_context: QuentContext, + ray_num_ranks: int, + ray_init_options: dict[str, Any], ) -> Iterator[StreamingEngine]: """ A streaming engine configured with a quent context from the 'quent_context' @@ -43,8 +45,14 @@ def engine_with_quent_context( pytest.importorskip("ray") import cudf_polars.engine.ray + # Always specify num_ranks: the default path sizes the engine from the + # GPUs Ray reports, which fails if this test shares an xdist worker + # with a test that already brought up the shared num_gpus=0 cluster. engine = cudf_polars.engine.ray.RayEngine( - executor_options={"quent_context": quent_context} + executor_options={"quent_context": quent_context}, + engine_options={"allow_gpu_sharing": True}, + ray_init_options=ray_init_options, + num_ranks=ray_num_ranks, ) elif backend == "dask": pytest.importorskip("distributed") diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index c1277f1489a4..dd5cc9a372e3 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ dependencies = [ "libkvikio==26.10.*,>=0.0.0a0", "librmm==26.10.*,>=0.0.0a0", - "nvidia-libnvcomp==5.2.0.13", + "nvidia-libnvcomp==5.3.0.16", "nvidia-nvjitlink>=13.3,<14", "rapids-logger==0.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -73,7 +73,7 @@ requires = [ "libkvikio==26.10.*,>=0.0.0a0", "librmm==26.10.*,>=0.0.0a0", "ninja", - "nvidia-libnvcomp==5.2.0.13", + "nvidia-libnvcomp==5.3.0.16", "nvidia-nvjitlink>=13.3,<14", "rapids-logger==0.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.