add support for Zoltan with PT-Scotch

- zoltan/CMakeLists.txt: add ENABLE_PARMETIS/ENABLE_PTSCOTCH flags.
- change include_dirs/link_libraries to depend on ENABLE_ flags.
- cmake/FindZoltan.cmake: remove default path when ZOLTAN_PREFIX is
  defined.
- search for pt-scotch or parmetis depending on what is enabled.
- zoltan/apfZoltan.h: add PTSCOTCH ZoltanMethod.
- zoltan/apfZoltanCallbacks.cc: make PARMETIS/PTSCOTCH options only work
  when support is compiled in.
- make GRAPH method prefer ParMETIS, then PT-Scotch, then PHG.
- guard PARMETIS-specifics.

Signed-off-by: Aiden Woodruff <[email protected]>

Author: bobpaw

cmake/FindZoltan.cmake

@@ -10,14 +10,28 @@ if(ZOLTAN_PREFIX)
   message(STATUS "ZOLTAN_PREFIX ${ZOLTAN_PREFIX}")
 endif()
 
-find_path(ZOLTAN_INCLUDE_DIR zoltan.h PATHS "${ZOLTAN_PREFIX}/include")
-
-find_library(ZOLTAN_LIBRARY zoltan PATHS "${ZOLTAN_PREFIX}/lib")
+if(DEFINED ZOLTAN_PREFIX)
+  find_path(ZOLTAN_INCLUDE_DIR zoltan.h
+    PATHS "${ZOLTAN_PREFIX}/include"
+    NO_DEFAULT_PATH
+  )
+  find_library(ZOLTAN_LIBRARY zoltan
+    PATHS "${ZOLTAN_PREFIX}/lib"
+    NO_DEFAULT_PATH
+  )
+else()
+  find_path(ZOLTAN_INCLUDE_DIR zoltan.h)
+  find_library(ZOLTAN_LIBRARY zoltan)
+endif()
 
 set(ZOLTAN_LIBRARIES ${ZOLTAN_LIBRARY} )
 set(ZOLTAN_INCLUDE_DIRS ${ZOLTAN_INCLUDE_DIR} )
 
-find_package(Parmetis MODULE REQUIRED)
+if(ENABLE_PTSCOTCH)
+  find_package(SCOTCH CONFIG REQUIRED)
+elseif(ENABLE_PARMETIS)
+  find_package(Parmetis MODULE REQUIRED)
+endif()
 
 include(FindPackageHandleStandardArgs)
 # handle the QUIETLY and REQUIRED arguments and set ZOLTAN_FOUND to TRUE

zoltan/CMakeLists.txt

@@ -5,9 +5,14 @@ endif()
 
 # Package options
 option(ENABLE_ZOLTAN "Enable Zoltan interface [ON|OFF]" OFF)
+option(ENABLE_PARMETIS "Enable ParMETIS for Zoltan [ON|OFF]" OFF)
+option(ENABLE_PTSCOTCH "Enable PT-Scotch for Zoltan [ON|OFF]" OFF)
 xsdk_add_tpl(ZOLTAN)
 xsdk_add_tpl(PARMETIS)
+xsdk_add_tpl(PTSCOTCH)
 message(STATUS "ENABLE_ZOLTAN: " ${ENABLE_ZOLTAN})
+message(STATUS "ENABLE_PARMETIS: " ${ENABLE_PARMETIS})
+message(STATUS "ENABLE_PTSCOTCH: " ${ENABLE_PTSCOTCH})
 
 if(SCOREC_NO_MPI AND ENABLE_ZOLTAN)
   message(FATAL_ERROR "SCOREC_NO_MPI is incompatible with Zoltan.")
@@ -58,15 +63,19 @@ target_link_libraries(apf_zoltan PUBLIC pcu apf)
 
 # Do extra work if zoltan is enabled
 if(ENABLE_ZOLTAN)
-  target_include_directories(apf_zoltan PRIVATE
-      ${ZOLTAN_INCLUDE_DIRS}
-      ${PARMETIS_INCLUDE_DIRS}
-      )
-  target_link_libraries(apf_zoltan PUBLIC
-    ${ZOLTAN_LIBRARIES}
-    ${PARMETIS_LIBRARIES}
-  )
+  target_include_directories(apf_zoltan PRIVATE ${ZOLTAN_INCLUDE_DIRS})
+  target_link_libraries(apf_zoltan PUBLIC ${ZOLTAN_LIBRARIES})
   target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_ZOLTAN)
+  if(ENABLE_PTSCOTCH)
+    target_link_libraries(apf_zoltan PUBLIC
+      SCOTCH::ptscotch SCOTCH::ptscotcherr
+    )
+    target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_PTSCOTCH)
+  elseif(ENABLE_PARMETIS)
+    target_include_directories(apf_zoltan PRIVATE ${PARMETIS_INCLUDE_DIRS})
+    target_link_libraries(apf_zoltan PUBLIC ${PARMETIS_LIBRARIES})
+    target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_PARMETIS)
+  endif()
 endif()
 
 scorec_export_library(apf_zoltan)

zoltan/apfZoltan.h

@@ -42,6 +42,7 @@ enum ZoltanMethod {
   HYPERGRAPH,
   /** \brief Use ParMetis */
   PARMETIS,
+  PTSCOTCH, /**< Use PT-Scotch */
   /** \brief General graph partitionig */
   GRAPH
 };

zoltan/apfZoltanCallbacks.cc

@@ -9,7 +9,9 @@
 #include "apfZoltanMesh.h"
 #include "apfZoltan.h"
 #include "apfShape.h"
+#ifdef PUMI_HAS_PARMETIS
 #include <metis.h>
+#endif
 #include <pcu_util.h>
 #include <lionPrint.h>
 #include <cstdlib>
@@ -29,9 +31,33 @@ static int setZoltanLbMethod(struct Zoltan_Struct* ztn, ZoltanMesh* zb)
     case HYPERGRAPH:
       lbMethod = "HYPERGRAPH"; break;
     case PARMETIS: //fall into GRAPH settings
+      lbMethod = "GRAPH";
+#ifdef PUMI_HAS_PARMETIS
+      Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "PARMETIS");
+#else
+      lion_oprint(1, "WARNING: ParMETIS ZoltanMethod requested but ParMETIS"
+        " was not enabled at build time.\n");
+#endif
+      break;
+    case PTSCOTCH:
+      lbMethod = "GRAPH";
+#ifdef PUMI_HAS_PTSCOTCH
+      Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "Scotch");
+#else
+      lion_oprint(1, "WARNING: PT-Scotch ZoltanMethod requested but PT-Scotch"
+        " was not enabled at build time.\n");
+#endif
+      break;
     case GRAPH:
       lbMethod = "GRAPH";
-      Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "PARMETIS"); // instead of PHG
+      // Prefer ParMETIS, then PT-Scotch, then Zoltan-native PHG.
+#if defined(PUMI_HAS_PARMETIS)
+      Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "PARMETIS");
+#elif defined(PUMI_HAS_PTSCOTCH)
+      Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "Scotch");
+#else
+      // Zoltan_Set_Param(ztn, "HYPERGRAPH_PACKAGE", "PHG"); //FIXME: not required?
+#endif
       break;
     default:
       lion_oprint(1,"ERROR %s Invalid LB_METHOD %d\n",__func__, zb->method);
@@ -68,8 +94,14 @@ static int setZoltanLbApproach(struct Zoltan_Struct* ztn, ZoltanMesh* zb)
       return 1;
   }
   Zoltan_Set_Param(ztn, "LB_APPROACH", ptnAp.c_str());
-  if ( (3 == zb->method) || (4 == zb->method) )
+  if (zb->method == PARMETIS
+#ifdef PUMI_HAS_PARMETIS // in this case GRAPH implies PARMETIS.
+    || zb->method == GRAPH
+#endif
+  ) {
     Zoltan_Set_Param(ztn, "PARMETIS_METHOD", pMethod.c_str());
+  }
+  // No zb->method == PTSCOTCH because Zoltan only supports RBISECT.
   return 0;
 }
 
@@ -304,7 +336,9 @@ ZoltanData::~ZoltanData()
 void ZoltanData::run()
 {
 /* ensure Metis indices are the same size as Zoltan indices */
+#ifdef PUMI_HAS_PARMETIS
   PCU_ALWAYS_ASSERT(IDXTYPEWIDTH == sizeof(ZOLTAN_ID_TYPE)*8);
+#endif
   setup();
   ptn();
 }
@@ -331,7 +365,9 @@ void ZoltanData::setup()
   if ( zb->isLocal && 0 != m->getPCU()->Self() )
     snprintf(paramStr, 128, "%d", 0);  //if local silence all but rank 0
   Zoltan_Set_Param(ztn, "debug_level", paramStr);
+#ifdef PUMI_HAS_PARMETIS
   Zoltan_Set_Param(ztn, "PARMETIS_OUTPUT_LEVEL", paramStr);
+#endif
   Zoltan_Set_Param(ztn, "CHECK_GRAPH", "0");
   Zoltan_Set_Param(ztn, "CHECK_HYPERGRAPH", "0");