GH-43296: [C++][FlightRPC] Remove Flight UCX transport (#43297)

### Rationale for this change

This was experimental and we should not encourage it's use.

### What changes are included in this PR?

Removal of UCX transport for Flight

### Are these changes tested?

Via CI

### Are there any user-facing changes?

Yes, we do remove UCX as a possible transport for Flight.

If users want something similar we should encourage the [Dissassociated IPC protocol implementation](https://arrow.apache.org/docs/dev/format/DissociatedIPC.html)
* GitHub Issue: #43296

Authored-by: Raúl Cumplido <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>

Author: raulcd

ci/docker/conda-cpp.dockerfile

@@ -37,8 +37,6 @@ RUN mamba install -q -y \
         doxygen \
         libnuma \
         python=${python} \
-        ucx \
-        ucx-proc=*=cpu \
         valgrind && \
     mamba clean --all
 

ci/scripts/cpp_build.sh

@@ -215,7 +215,6 @@ else
     -DARROW_WITH_OPENTELEMETRY=${ARROW_WITH_OPENTELEMETRY:-OFF} \
     -DARROW_WITH_MUSL=${ARROW_WITH_MUSL:-OFF} \
     -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY:-OFF} \
-    -DARROW_WITH_UCX=${ARROW_WITH_UCX:-OFF} \
     -DARROW_WITH_UTF8PROC=${ARROW_WITH_UTF8PROC:-ON} \
     -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \
     -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \

cpp/cmake_modules/DefineOptions.cmake

@@ -539,10 +539,6 @@ takes precedence over ccache if a storage backend is configured" ON)
   define_option(ARROW_WITH_ZLIB "Build with zlib compression" OFF)
   define_option(ARROW_WITH_ZSTD "Build with zstd compression" OFF)
 
-  define_option(ARROW_WITH_UCX
-                "Build with UCX transport for Arrow Flight;(only used if ARROW_FLIGHT is ON)"
-                OFF)
-
   define_option(ARROW_WITH_UTF8PROC
                 "Build with support for Unicode properties using the utf8proc library;(only used if ARROW_COMPUTE is ON or ARROW_GANDIVA is ON)"
                 ON)

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -67,7 +67,6 @@ set(ARROW_THIRDPARTY_DEPENDENCIES
     Snappy
     Substrait
     Thrift
-    ucx
     utf8proc
     xsimd
     ZLIB
@@ -218,8 +217,6 @@ macro(build_dependency DEPENDENCY_NAME)
     build_substrait()
   elseif("${DEPENDENCY_NAME}" STREQUAL "Thrift")
     build_thrift()
-  elseif("${DEPENDENCY_NAME}" STREQUAL "ucx")
-    build_ucx()
   elseif("${DEPENDENCY_NAME}" STREQUAL "utf8proc")
     build_utf8proc()
   elseif("${DEPENDENCY_NAME}" STREQUAL "xsimd")
@@ -821,13 +818,6 @@ else()
   )
 endif()
 
-if(DEFINED ENV{ARROW_UCX_URL})
-  set(ARROW_UCX_SOURCE_URL "$ENV{ARROW_UCX_URL}")
-else()
-  set_urls(ARROW_UCX_SOURCE_URL
-           "https://github.com/openucx/ucx/archive/v${ARROW_UCX_BUILD_VERSION}.tar.gz")
-endif()
-
 if(DEFINED ENV{ARROW_UTF8PROC_URL})
   set(ARROW_UTF8PROC_SOURCE_URL "$ENV{ARROW_UTF8PROC_URL}")
 else()
@@ -5548,85 +5538,5 @@ if(ARROW_WITH_AZURE_SDK)
   set(AZURE_SDK_LINK_LIBRARIES Azure::azure-storage-files-datalake
                                Azure::azure-storage-blobs Azure::azure-identity)
 endif()
-# ----------------------------------------------------------------------
-# ucx - communication framework for modern, high-bandwidth and low-latency networks
-
-macro(build_ucx)
-  message(STATUS "Building UCX from source")
-
-  set(UCX_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/ucx_ep-install")
-
-  # link with static ucx libraries leads to test failures, use shared libs instead
-  set(UCX_SHARED_LIB_UCP "${UCX_PREFIX}/lib/libucp${CMAKE_SHARED_LIBRARY_SUFFIX}")
-  set(UCX_SHARED_LIB_UCT "${UCX_PREFIX}/lib/libuct${CMAKE_SHARED_LIBRARY_SUFFIX}")
-  set(UCX_SHARED_LIB_UCS "${UCX_PREFIX}/lib/libucs${CMAKE_SHARED_LIBRARY_SUFFIX}")
-  set(UCX_SHARED_LIB_UCM "${UCX_PREFIX}/lib/libucm${CMAKE_SHARED_LIBRARY_SUFFIX}")
-
-  set(UCX_CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure)
-  list(APPEND
-       UCX_CONFIGURE_COMMAND
-       "CC=${CMAKE_C_COMPILER}"
-       "CXX=${CMAKE_CXX_COMPILER}"
-       "CFLAGS=${EP_C_FLAGS}"
-       "CXXFLAGS=${EP_CXX_FLAGS}"
-       "--prefix=${UCX_PREFIX}"
-       "--enable-mt"
-       "--enable-shared")
-  if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
-    list(APPEND
-         UCX_CONFIGURE_COMMAND
-         "--enable-profiling"
-         "--enable-frame-pointer"
-         "--enable-stats"
-         "--enable-fault-injection"
-         "--enable-debug-data")
-  else()
-    list(APPEND
-         UCX_CONFIGURE_COMMAND
-         "--disable-logging"
-         "--disable-debug"
-         "--disable-assertions"
-         "--disable-params-check")
-  endif()
-  set(UCX_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS})
-  externalproject_add(ucx_ep
-                      ${EP_COMMON_OPTIONS}
-                      URL ${ARROW_UCX_SOURCE_URL}
-                      URL_HASH "SHA256=${ARROW_UCX_BUILD_SHA256_CHECKSUM}"
-                      CONFIGURE_COMMAND ${UCX_CONFIGURE_COMMAND}
-                      BUILD_IN_SOURCE 1
-                      BUILD_COMMAND ${UCX_BUILD_COMMAND}
-                      BUILD_BYPRODUCTS "${UCX_SHARED_LIB_UCP}" "${UCX_SHARED_LIB_UCT}"
-                                       "${UCX_SHARED_LIB_UCS}" "${UCX_SHARED_LIB_UCM}"
-                      INSTALL_COMMAND ${MAKE} install)
-
-  # ucx cmake module sets UCX_INCLUDE_DIRS
-  set(UCX_INCLUDE_DIRS "${UCX_PREFIX}/include")
-  file(MAKE_DIRECTORY "${UCX_INCLUDE_DIRS}")
-
-  add_library(ucx::ucp SHARED IMPORTED)
-  set_target_properties(ucx::ucp PROPERTIES IMPORTED_LOCATION "${UCX_SHARED_LIB_UCP}")
-  add_library(ucx::uct SHARED IMPORTED)
-  set_target_properties(ucx::uct PROPERTIES IMPORTED_LOCATION "${UCX_SHARED_LIB_UCT}")
-  add_library(ucx::ucs SHARED IMPORTED)
-  set_target_properties(ucx::ucs PROPERTIES IMPORTED_LOCATION "${UCX_SHARED_LIB_UCS}")
-
-  add_dependencies(ucx::ucp ucx_ep)
-  add_dependencies(ucx::uct ucx_ep)
-  add_dependencies(ucx::ucs ucx_ep)
-endmacro()
-
-if(ARROW_WITH_UCX)
-  resolve_dependency(ucx
-                     ARROW_CMAKE_PACKAGE_NAME
-                     ArrowFlight
-                     ARROW_PC_PACKAGE_NAME
-                     arrow-flight
-                     PC_PACKAGE_NAMES
-                     ucx)
-  add_library(ucx::ucx INTERFACE IMPORTED)
-  target_include_directories(ucx::ucx INTERFACE "${UCX_INCLUDE_DIRS}")
-  target_link_libraries(ucx::ucx INTERFACE ucx::ucp ucx::uct ucx::ucs)
-endif()
 
 message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}")

cpp/src/arrow/flight/CMakeLists.txt

@@ -332,21 +332,8 @@ if(ARROW_BUILD_BENCHMARKS)
 
   add_dependencies(arrow_flight arrow-flight-benchmark)
 
-  if(ARROW_WITH_UCX)
-    if(ARROW_FLIGHT_TEST_LINKAGE STREQUAL "static")
-      target_link_libraries(arrow-flight-benchmark arrow_flight_transport_ucx_static)
-      target_link_libraries(arrow-flight-perf-server arrow_flight_transport_ucx_static)
-    else()
-      target_link_libraries(arrow-flight-benchmark arrow_flight_transport_ucx_shared)
-      target_link_libraries(arrow-flight-perf-server arrow_flight_transport_ucx_shared)
-    endif()
-  endif()
 endif(ARROW_BUILD_BENCHMARKS)
 
-if(ARROW_WITH_UCX)
-  add_subdirectory(transport/ucx)
-endif()
-
 if(ARROW_FLIGHT_SQL)
   add_subdirectory(sql)
 

cpp/src/arrow/flight/flight_benchmark.cc

@@ -43,17 +43,10 @@
 #  include <cuda.h>
 #  include "arrow/gpu/cuda_api.h"
 #endif
-#ifdef ARROW_WITH_UCX
-#  include "arrow/flight/transport/ucx/ucx.h"
-#endif
 
 DEFINE_bool(cuda, false, "Allocate results in CUDA memory");
 DEFINE_string(transport, "grpc",
-              "The network transport to use. Supported: \"grpc\" (default)"
-#ifdef ARROW_WITH_UCX
-              ", \"ucx\""
-#endif  // ARROW_WITH_UCX
-              ".");
+              "The network transport to use. Supported: \"grpc\" (default).");
 DEFINE_string(server_host, "",
               "An existing performance server to benchmark against (leave blank to spawn "
               "one automatically)");
@@ -506,21 +499,6 @@ int main(int argc, char** argv) {
         options.disable_server_verification = true;
       }
     }
-  } else if (FLAGS_transport == "ucx") {
-#ifdef ARROW_WITH_UCX
-    arrow::flight::transport::ucx::InitializeFlightUcx();
-    if (FLAGS_test_unix || !FLAGS_server_unix.empty()) {
-      std::cerr << "Transport does not support domain sockets: " << FLAGS_transport
-                << std::endl;
-      return EXIT_FAILURE;
-    }
-    ARROW_CHECK_OK(arrow::flight::Location::Parse("ucx://" + FLAGS_server_host + ":" +
-                                                  std::to_string(FLAGS_server_port))
-                       .Value(&location));
-#else
-    std::cerr << "Not built with transport: " << FLAGS_transport << std::endl;
-    return EXIT_FAILURE;
-#endif
   } else {
     std::cerr << "Unknown transport: " << FLAGS_transport << std::endl;
     return EXIT_FAILURE;

cpp/src/arrow/flight/perf_server.cc

@@ -44,17 +44,10 @@
 #ifdef ARROW_CUDA
 #  include "arrow/gpu/cuda_api.h"
 #endif
-#ifdef ARROW_WITH_UCX
-#  include "arrow/flight/transport/ucx/ucx.h"
-#endif
 
 DEFINE_bool(cuda, false, "Allocate results in CUDA memory");
 DEFINE_string(transport, "grpc",
-              "The network transport to use. Supported: \"grpc\" (default)"
-#ifdef ARROW_WITH_UCX
-              ", \"ucx\""
-#endif  // ARROW_WITH_UCX
-              ".");
+              "The network transport to use. Supported: \"grpc\" (default).");
 DEFINE_string(server_host, "localhost", "Host where the server is running on");
 DEFINE_int32(port, 31337, "Server port to listen on");
 DEFINE_string(server_unix, "", "Unix socket path where the server is running on");
@@ -280,29 +273,6 @@ int main(int argc, char** argv) {
       ARROW_CHECK_OK(arrow::flight::Location::ForGrpcUnix(FLAGS_server_unix)
                          .Value(&connect_location));
     }
-  } else if (FLAGS_transport == "ucx") {
-#ifdef ARROW_WITH_UCX
-    arrow::flight::transport::ucx::InitializeFlightUcx();
-    if (FLAGS_server_unix.empty()) {
-      if (!FLAGS_cert_file.empty() || !FLAGS_key_file.empty()) {
-        std::cerr << "Transport does not support TLS: " << FLAGS_transport << std::endl;
-        return EXIT_FAILURE;
-      }
-      ARROW_CHECK_OK(arrow::flight::Location::Parse("ucx://" + FLAGS_server_host + ":" +
-                                                    std::to_string(FLAGS_port))
-                         .Value(&bind_location));
-      ARROW_CHECK_OK(arrow::flight::Location::Parse("ucx://" + FLAGS_server_host + ":" +
-                                                    std::to_string(FLAGS_port))
-                         .Value(&connect_location));
-    } else {
-      std::cerr << "Transport does not support domain sockets: " << FLAGS_transport
-                << std::endl;
-      return EXIT_FAILURE;
-    }
-#else
-    std::cerr << "Not built with transport: " << FLAGS_transport << std::endl;
-    return EXIT_FAILURE;
-#endif
   } else {
     std::cerr << "Unknown transport: " << FLAGS_transport << std::endl;
     return EXIT_FAILURE;