From 41fe02832614c49f99c1cdbcd966f34a69426af7 Mon Sep 17 00:00:00 2001 From: Aiden Woodruff Date: Mon, 31 Mar 2025 00:42:29 -0400 Subject: [PATCH 1/2] add support for Zoltan with PT-Scotch - zoltan/CMakeLists.txt: add ENABLE_PARMETIS/ENABLE_PTSCOTCH flags. - change include_dirs/link_libraries to depend on ENABLE_ flags. - cmake/FindZoltan.cmake: remove default path when ZOLTAN_PREFIX is defined. - search for pt-scotch or parmetis depending on what is enabled. - zoltan/apfZoltan.h: add PTSCOTCH ZoltanMethod. - zoltan/apfZoltanCallbacks.cc: make PARMETIS/PTSCOTCH options only work when support is compiled in. - make GRAPH method prefer ParMETIS, then PT-Scotch, then PHG. - guard PARMETIS-specifics. Signed-off-by: Aiden Woodruff --- cmake/FindZoltan.cmake | 22 ++++++++++++++++---- zoltan/CMakeLists.txt | 25 ++++++++++++++-------- zoltan/apfZoltan.h | 1 + zoltan/apfZoltanCallbacks.cc | 40 ++++++++++++++++++++++++++++++++++-- 4 files changed, 74 insertions(+), 14 deletions(-) diff --git a/cmake/FindZoltan.cmake b/cmake/FindZoltan.cmake index 24d8ad654..0d1a2bc59 100644 --- a/cmake/FindZoltan.cmake +++ b/cmake/FindZoltan.cmake @@ -10,14 +10,28 @@ if(ZOLTAN_PREFIX) message(STATUS "ZOLTAN_PREFIX ${ZOLTAN_PREFIX}") endif() -find_path(ZOLTAN_INCLUDE_DIR zoltan.h PATHS "${ZOLTAN_PREFIX}/include") - -find_library(ZOLTAN_LIBRARY zoltan PATHS "${ZOLTAN_PREFIX}/lib") +if(DEFINED ZOLTAN_PREFIX) + find_path(ZOLTAN_INCLUDE_DIR zoltan.h + PATHS "${ZOLTAN_PREFIX}/include" + NO_DEFAULT_PATH + ) + find_library(ZOLTAN_LIBRARY zoltan + PATHS "${ZOLTAN_PREFIX}/lib" + NO_DEFAULT_PATH + ) +else() + find_path(ZOLTAN_INCLUDE_DIR zoltan.h) + find_library(ZOLTAN_LIBRARY zoltan) +endif() set(ZOLTAN_LIBRARIES ${ZOLTAN_LIBRARY} ) set(ZOLTAN_INCLUDE_DIRS ${ZOLTAN_INCLUDE_DIR} ) -find_package(Parmetis MODULE REQUIRED) +if(ENABLE_PTSCOTCH) + find_package(SCOTCH CONFIG REQUIRED) +elseif(ENABLE_PARMETIS) + find_package(Parmetis MODULE REQUIRED) +endif() include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set ZOLTAN_FOUND to TRUE diff --git a/zoltan/CMakeLists.txt b/zoltan/CMakeLists.txt index 92e9638a2..d51f6e7f5 100644 --- a/zoltan/CMakeLists.txt +++ b/zoltan/CMakeLists.txt @@ -5,9 +5,14 @@ endif() # Package options option(ENABLE_ZOLTAN "Enable Zoltan interface [ON|OFF]" OFF) +option(ENABLE_PARMETIS "Enable ParMETIS for Zoltan [ON|OFF]" OFF) +option(ENABLE_PTSCOTCH "Enable PT-Scotch for Zoltan [ON|OFF]" OFF) xsdk_add_tpl(ZOLTAN) xsdk_add_tpl(PARMETIS) +xsdk_add_tpl(PTSCOTCH) message(STATUS "ENABLE_ZOLTAN: " ${ENABLE_ZOLTAN}) +message(STATUS "ENABLE_PARMETIS: " ${ENABLE_PARMETIS}) +message(STATUS "ENABLE_PTSCOTCH: " ${ENABLE_PTSCOTCH}) if(SCOREC_NO_MPI AND ENABLE_ZOLTAN) message(FATAL_ERROR "SCOREC_NO_MPI is incompatible with Zoltan.") @@ -58,15 +63,19 @@ target_link_libraries(apf_zoltan PUBLIC pcu apf) # Do extra work if zoltan is enabled if(ENABLE_ZOLTAN) - target_include_directories(apf_zoltan PRIVATE - ${ZOLTAN_INCLUDE_DIRS} - ${PARMETIS_INCLUDE_DIRS} - ) - target_link_libraries(apf_zoltan PUBLIC - ${ZOLTAN_LIBRARIES} - ${PARMETIS_LIBRARIES} - ) + target_include_directories(apf_zoltan PRIVATE ${ZOLTAN_INCLUDE_DIRS}) + target_link_libraries(apf_zoltan PUBLIC ${ZOLTAN_LIBRARIES}) target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_ZOLTAN) + if(ENABLE_PTSCOTCH) + target_link_libraries(apf_zoltan PUBLIC + SCOTCH::ptscotch SCOTCH::ptscotcherr + ) + target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_PTSCOTCH) + elseif(ENABLE_PARMETIS) + target_include_directories(apf_zoltan PRIVATE ${PARMETIS_INCLUDE_DIRS}) + target_link_libraries(apf_zoltan PUBLIC ${PARMETIS_LIBRARIES}) + target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_PARMETIS) + endif() endif() scorec_export_library(apf_zoltan) diff --git a/zoltan/apfZoltan.h b/zoltan/apfZoltan.h index 9c50742b4..56117ecbe 100644 --- a/zoltan/apfZoltan.h +++ b/zoltan/apfZoltan.h @@ -42,6 +42,7 @@ enum ZoltanMethod { HYPERGRAPH, /** \brief Use ParMetis */ PARMETIS, + PTSCOTCH, /**< Use PT-Scotch */ /** \brief General graph partitionig */ GRAPH }; diff --git a/zoltan/apfZoltanCallbacks.cc b/zoltan/apfZoltanCallbacks.cc index 7fe724fc5..9df989409 100644 --- a/zoltan/apfZoltanCallbacks.cc +++ b/zoltan/apfZoltanCallbacks.cc @@ -9,7 +9,9 @@ #include "apfZoltanMesh.h" #include "apfZoltan.h" #include "apfShape.h" +#ifdef PUMI_HAS_PARMETIS #include +#endif #include #include #include @@ -29,9 +31,33 @@ static int setZoltanLbMethod(struct Zoltan_Struct* ztn, ZoltanMesh* zb) case HYPERGRAPH: lbMethod = "HYPERGRAPH"; break; case PARMETIS: //fall into GRAPH settings + lbMethod = "GRAPH"; +#ifdef PUMI_HAS_PARMETIS + Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "PARMETIS"); +#else + lion_oprint(1, "WARNING: ParMETIS ZoltanMethod requested but ParMETIS" + " was not enabled at build time.\n"); +#endif + break; + case PTSCOTCH: + lbMethod = "GRAPH"; +#ifdef PUMI_HAS_PTSCOTCH + Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "Scotch"); +#else + lion_oprint(1, "WARNING: PT-Scotch ZoltanMethod requested but PT-Scotch" + " was not enabled at build time.\n"); +#endif + break; case GRAPH: lbMethod = "GRAPH"; - Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "PARMETIS"); // instead of PHG + // Prefer ParMETIS, then PT-Scotch, then Zoltan-native PHG. +#if defined(PUMI_HAS_PARMETIS) + Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "PARMETIS"); +#elif defined(PUMI_HAS_PTSCOTCH) + Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "Scotch"); +#else + // Zoltan_Set_Param(ztn, "HYPERGRAPH_PACKAGE", "PHG"); //FIXME: not required? +#endif break; default: lion_oprint(1,"ERROR %s Invalid LB_METHOD %d\n",__func__, zb->method); @@ -68,8 +94,14 @@ static int setZoltanLbApproach(struct Zoltan_Struct* ztn, ZoltanMesh* zb) return 1; } Zoltan_Set_Param(ztn, "LB_APPROACH", ptnAp.c_str()); - if ( (3 == zb->method) || (4 == zb->method) ) + if (zb->method == PARMETIS +#ifdef PUMI_HAS_PARMETIS // in this case GRAPH implies PARMETIS. + || zb->method == GRAPH +#endif + ) { Zoltan_Set_Param(ztn, "PARMETIS_METHOD", pMethod.c_str()); + } + // No zb->method == PTSCOTCH because Zoltan only supports RBISECT. return 0; } @@ -304,7 +336,9 @@ ZoltanData::~ZoltanData() void ZoltanData::run() { /* ensure Metis indices are the same size as Zoltan indices */ +#ifdef PUMI_HAS_PARMETIS PCU_ALWAYS_ASSERT(IDXTYPEWIDTH == sizeof(ZOLTAN_ID_TYPE)*8); +#endif setup(); ptn(); } @@ -331,7 +365,9 @@ void ZoltanData::setup() if ( zb->isLocal && 0 != m->getPCU()->Self() ) snprintf(paramStr, 128, "%d", 0); //if local silence all but rank 0 Zoltan_Set_Param(ztn, "debug_level", paramStr); +#ifdef PUMI_HAS_PARMETIS Zoltan_Set_Param(ztn, "PARMETIS_OUTPUT_LEVEL", paramStr); +#endif Zoltan_Set_Param(ztn, "CHECK_GRAPH", "0"); Zoltan_Set_Param(ztn, "CHECK_HYPERGRAPH", "0"); From ef1377082f90a773a68c44838c2f4d49bbdf1046 Mon Sep 17 00:00:00 2001 From: Aiden Woodruff Date: Tue, 1 Apr 2025 13:30:59 -0400 Subject: [PATCH 2/2] detect required Zoltan dependency - cmake/FindZoltan.cmake: fix find_path/find_library when ZOLTAN_PREFIX not specified. since it is always defined, check instead for truthiness. - add FATAL_ERROR if Zoltan is not found. - add debug message with found Zoltan location. - find Zoltan_config.h and search for #define HAVE_PARMETIS and #define HAVE_SCOTCH. use the results to determine which libraries are required. - set the PUMI_HAS_PARMETIS and PUMI_HAS_PTSCOTCH from this file. - zoltan/CMakeLists.txt: remove ENABLE_PARMETIS and ENABLE_PTSCOTCH cache options. - replace ENABLE_PTSCOTCH/ENABLE_PARMETIS with PUMI_HAS_PARMETIS/PUMI_HAS_PTSCOTCH which are detected in FindZoltan.cmake. Signed-off-by: Aiden Woodruff --- cmake/FindZoltan.cmake | 33 ++++++++++++++++++++++++++++----- zoltan/CMakeLists.txt | 11 +++-------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/cmake/FindZoltan.cmake b/cmake/FindZoltan.cmake index 0d1a2bc59..34e556da7 100644 --- a/cmake/FindZoltan.cmake +++ b/cmake/FindZoltan.cmake @@ -10,7 +10,7 @@ if(ZOLTAN_PREFIX) message(STATUS "ZOLTAN_PREFIX ${ZOLTAN_PREFIX}") endif() -if(DEFINED ZOLTAN_PREFIX) +if(ZOLTAN_PREFIX) find_path(ZOLTAN_INCLUDE_DIR zoltan.h PATHS "${ZOLTAN_PREFIX}/include" NO_DEFAULT_PATH @@ -23,14 +23,37 @@ else() find_path(ZOLTAN_INCLUDE_DIR zoltan.h) find_library(ZOLTAN_LIBRARY zoltan) endif() +if (ZOLTAN_INCLUDE_DIR) + message(DEBUG "ZOLTAN_INCLUDE_DIR ${ZOLTAN_INCLUDE_DIR}") +else() + message(FATAL_ERROR "Zoltan not found.") +endif() set(ZOLTAN_LIBRARIES ${ZOLTAN_LIBRARY} ) set(ZOLTAN_INCLUDE_DIRS ${ZOLTAN_INCLUDE_DIR} ) -if(ENABLE_PTSCOTCH) - find_package(SCOTCH CONFIG REQUIRED) -elseif(ENABLE_PARMETIS) +find_file(ZOLTAN_CONFIG_FILE Zoltan_config.h + "${ZOLTAN_INCLUDE_DIRS}" + NO_DEFAULT_PATH +) +message(DEBUG "ZOLTAN_CONFIG_FILE ${ZOLTAN_CONFIG_FILE}") +file(READ "${ZOLTAN_CONFIG_FILE}" ZOLTAN_CONFIG_TEXT) +string(FIND + "${ZOLTAN_CONFIG_TEXT}" "#define HAVE_PARMETIS" + ZOLTAN_FIND_PARMETIS +) +string(FIND + "${ZOLTAN_CONFIG_TEXT}" "#define HAVE_SCOTCH" + ZOLTAN_FIND_PTSCOTCH +) + +if(NOT ZOLTAN_FIND_PARMETIS EQUAL -1) find_package(Parmetis MODULE REQUIRED) + set(PUMI_HAS_PARMETIS TRUE) +endif() +if(NOT ZOLTAN_FIND_PTSCOTCH EQUAL -1) + find_package(SCOTCH CONFIG REQUIRED) + set(PUMI_HAS_PTSCOTCH TRUE) endif() include(FindPackageHandleStandardArgs) @@ -42,4 +65,4 @@ find_package_handle_standard_args( ZOLTAN_LIBRARY ZOLTAN_INCLUDE_DIR ) -mark_as_advanced(ZOLTAN_INCLUDE_DIR ZOLTAN_LIBRARY ) +mark_as_advanced(ZOLTAN_INCLUDE_DIR ZOLTAN_LIBRARY ZOLTAN_CONFIG_FILE) diff --git a/zoltan/CMakeLists.txt b/zoltan/CMakeLists.txt index d51f6e7f5..92c929c42 100644 --- a/zoltan/CMakeLists.txt +++ b/zoltan/CMakeLists.txt @@ -5,14 +5,8 @@ endif() # Package options option(ENABLE_ZOLTAN "Enable Zoltan interface [ON|OFF]" OFF) -option(ENABLE_PARMETIS "Enable ParMETIS for Zoltan [ON|OFF]" OFF) -option(ENABLE_PTSCOTCH "Enable PT-Scotch for Zoltan [ON|OFF]" OFF) xsdk_add_tpl(ZOLTAN) -xsdk_add_tpl(PARMETIS) -xsdk_add_tpl(PTSCOTCH) message(STATUS "ENABLE_ZOLTAN: " ${ENABLE_ZOLTAN}) -message(STATUS "ENABLE_PARMETIS: " ${ENABLE_PARMETIS}) -message(STATUS "ENABLE_PTSCOTCH: " ${ENABLE_PTSCOTCH}) if(SCOREC_NO_MPI AND ENABLE_ZOLTAN) message(FATAL_ERROR "SCOREC_NO_MPI is incompatible with Zoltan.") @@ -66,12 +60,13 @@ if(ENABLE_ZOLTAN) target_include_directories(apf_zoltan PRIVATE ${ZOLTAN_INCLUDE_DIRS}) target_link_libraries(apf_zoltan PUBLIC ${ZOLTAN_LIBRARIES}) target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_ZOLTAN) - if(ENABLE_PTSCOTCH) + if(PUMI_HAS_PTSCOTCH) target_link_libraries(apf_zoltan PUBLIC SCOTCH::ptscotch SCOTCH::ptscotcherr ) target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_PTSCOTCH) - elseif(ENABLE_PARMETIS) + endif() + if(PUMI_HAS_PARMETIS) target_include_directories(apf_zoltan PRIVATE ${PARMETIS_INCLUDE_DIRS}) target_link_libraries(apf_zoltan PUBLIC ${PARMETIS_LIBRARIES}) target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_PARMETIS)