PCU: Add nompi stub

.github/workflows/cmake.yml

@@ -15,6 +15,7 @@ jobs:
           - { name: GNU, CC: gcc-10, CXX: g++-10 }
           - { name: LLVM, CC: clang, CXX: clang++ }
         build_type: [Debug, Release]
+        no_mpi: [OFF, ON]
 
     steps:
     - uses: actions/checkout@v4
@@ -30,7 +31,7 @@ jobs:
       env:
         MPICH_CXX: ${{matrix.compiler.CXX}}
         MPICH_CC: ${{matrix.compiler.CC}}
-      run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install
+      run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install -DSCOREC_NO_MPI=${{matrix.no_mpi}}
 
     - name: Build
       env:
@@ -54,3 +55,20 @@ jobs:
       run: |
         cmake -S ${{github.workspace}}/doc -B ${{github.workspace}}/buildExample -DCMAKE_CXX_COMPILER=mpicxx -DSCOREC_PREFIX=${{github.workspace}}/build/install
         cmake --build ${{github.workspace}}/buildExample
+
+    - name: Build MPI-NoMPI Example
+      # Test if a SCOREC_NO_MPI build works with MPI applications.
+      if: >-
+        matrix.compiler.name == 'GNU' && matrix.build_type == 'Release' &&
+        matrix.no_mpi == 'ON'
+      env:
+        MPICH_CXX: ${{matrix.compiler.CXX}}
+        MPICH_CC: ${{matrix.compiler.CC}}
+      run: >
+        cmake -S ${{github.workspace}}/example/mpi-nompi
+        -B ${{github.workspace}}/example/mpi-nompi/build
+        -DCMAKE_CXX_COMPILER=mpicxx
+        -DSCOREC_PREFIX=${{github.workspace}}/build/install ;
+        cmake --build ${{github.workspace}}/example/mpi-nompi/build ;
+        ctest --test-dir ${{github.workspace}}/example/mpi-nompi/build
+        --output-on-failure

CMakeLists.txt

@@ -6,7 +6,7 @@ endif()
 # This is the top level CMake file for the SCOREC build
 cmake_minimum_required(VERSION 3.8)
 
-project(SCOREC VERSION 3.0.3 LANGUAGES CXX C)
+project(SCOREC VERSION 4.0.0 LANGUAGES CXX C)
 
 include(cmake/bob.cmake)
 include(cmake/xsdk.cmake)
@@ -29,6 +29,15 @@ else()
   bob_set_cxx_standard(14)
 endif()
 
+#
+# option related to MPI
+#
+option(SCOREC_NO_MPI "Disable MPI support.  When enabled only serial execution is supported." OFF)
+message(STATUS "SCOREC_NO_MPI: ${SCOREC_NO_MPI}")
+if(SCOREC_NO_MPI AND ENABLE_CGNS)
+  message(FATAL_ERROR "SCOREC_NO_MPI is incompatible with CGNS.")
+endif()
+
 # Set some default compiler flags that should always be used
 if(NOT USE_XSDK_DEFAULTS)
   bob_set_shared_libs()
@@ -133,7 +142,10 @@ endif()
 
 macro(scorec_export_library target)
 bob_export_target(${target})
-install(FILES ${HEADERS} DESTINATION include)
+install(FILES ${HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
+target_include_directories(${target} INTERFACE
+  "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
+)
 endmacro(scorec_export_library)
 
 if(ENABLE_SIMMETRIX)
@@ -162,6 +174,10 @@ if(ENABLE_CGNS)
   add_definitions(-DHAVE_CGNS)
 endif()
 
+configure_file(SCOREC_config.h.in SCOREC_config.h)
+install(FILES "${CMAKE_BINARY_DIR}/SCOREC_config.h" DESTINATION include)
+include_directories(PUBLIC "$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}>")
+
 # Include the SCOREC project packages
 add_subdirectory(lion)
 add_subdirectory(pcu)

SCOREC_config.h.in

@@ -0,0 +1,7 @@
+#ifndef SCOREC_CONFIG_H
+#define SCOREC_CONFIG_H
+
+#cmakedefine SCOREC_NO_MPI
+
+#endif // SCOREC_CONFIG_H
+

apf/CMakeLists.txt

@@ -93,7 +93,6 @@ add_library(apf ${SOURCES})
 # Include directories
 target_include_directories(apf PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 # Link this library to these others

apf/apfCGNS.cc

@@ -545,8 +545,9 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe
         cgp_error_exit();
 
       std::vector<int> allNumbersForThisType(m->getPCU()->Peers(), 0);
-      MPI_Allgather(&numbersByElementType[o], 1, MPI_INT, allNumbersForThisType.data(), 1,
-                    MPI_INT, m->getPCU()->GetMPIComm());
+      m->getPCU()->Allgather(
+        &numbersByElementType[o], allNumbersForThisType.data(), 1
+      );
 
       cgsize_t num = 0;
       for (int i = 0; i < m->getPCU()->Self(); i++)
@@ -656,8 +657,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
         }
 
         std::vector<int> allNumbersForThisType(m->getPCU()->Peers(), 0);
-        MPI_Allgather(&number, 1, MPI_INT, allNumbersForThisType.data(), 1,
-                      MPI_INT, m->getPCU()->GetMPIComm());
+        m->getPCU()->Allgather(&number, allNumbersForThisType.data(), 1);
 
         cgsize_t num = 0;
         for (int i = 0; i < m->getPCU()->Self(); i++)
@@ -684,19 +684,16 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
       }
     }
     std::vector<int> cacheStarts(m->getPCU()->Peers(), 0);
-    MPI_Allgather(&cacheStart, 1, MPI_INT, cacheStarts.data(), 1,
-                  MPI_INT, m->getPCU()->GetMPIComm());
+    m->getPCU()->Allgather(&cacheStart, cacheStarts.data(), 1);
     std::vector<int> cacheEnds(m->getPCU()->Peers(), 0);
-    MPI_Allgather(&cacheEnd, 1, MPI_INT, cacheEnds.data(), 1,
-                  MPI_INT, m->getPCU()->GetMPIComm());
+    m->getPCU()->Allgather(&cacheEnd, cacheEnds.data(), 1);
     return std::make_pair(cacheStarts, cacheEnds);
   };
 
   const auto globalElementList = [&m](const std::vector<cgsize_t> &bcList, std::vector<cgsize_t> &allElements) {
     std::vector<int> sizes(m->getPCU()->Peers(), 0); // important initialiser
     const int l = bcList.size();
-    MPI_Allgather(&l, 1, MPI_INT, sizes.data(), 1,
-                  MPI_INT, m->getPCU()->GetMPIComm());
+    m->getPCU()->Allgather(&l, sizes.data(), 1);
 
     int totalLength = 0;
     for (const auto &i : sizes)
@@ -708,9 +705,15 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
       displacement[i] = displacement[i - 1] + sizes[i - 1];
 
     allElements.resize(totalLength);
+    #ifndef SCOREC_NO_MPI
+    PCU_Comm comm;
+    m->getPCU()->DupComm(&comm);
     MPI_Allgatherv(bcList.data(), bcList.size(), MPI_INT, allElements.data(),
-                   sizes.data(), displacement.data(), MPI_INT,
-                   m->getPCU()->GetMPIComm());
+                   sizes.data(), displacement.data(), MPI_INT, comm);
+    MPI_Comm_free(&comm);
+    #else
+    std::copy(bcList.begin(), bcList.end(), allElements.begin());
+    #endif
   };
 
   const auto doVertexBC = [&](const auto &iter) {
@@ -1046,7 +1049,8 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap
   sizes[2] = 0;                 // nodes are unsorted, as defined by api
 
   // Copy communicator
-  auto communicator = m->getPCU()->GetMPIComm();
+  PCU_Comm communicator;
+  m->getPCU()->DupComm(&communicator);
   cgp_mpi_comm(communicator);
   //
   cgp_pio_mode(CGP_INDEPENDENT);
@@ -1134,6 +1138,9 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap
   destroyGlobalNumbering(gcn);
   //
   cgp_close(cgns.index);
+  #ifndef SCOREC_NO_MPI
+  MPI_Comm_free(&communicator);
+  #endif
 }
 } // namespace
 

apf_cap/CMakeLists.txt

@@ -39,7 +39,6 @@ include(GNUInstallDirs)
 
 target_include_directories(apf_cap PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
 )
 
 scorec_export_library(apf_cap)

apf_sim/CMakeLists.txt

@@ -19,7 +19,6 @@ target_link_libraries(apf_sim PUBLIC ${SIMMODSUITE_LIBS})
 target_include_directories(gmi_sim PUBLIC ${SIMMODSUITE_INCLUDE_DIR})
 target_include_directories(apf_sim PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 option(ENABLE_FIELDSIM "Enable use of FieldSim from Simmetrix SimModSuite" FALSE)

can/CMakeLists.txt

@@ -11,7 +11,6 @@ add_library(can INTERFACE)
 # Include directories
 target_include_directories(can INTERFACE
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 scorec_export_library(can)

crv/CMakeLists.txt

@@ -33,7 +33,6 @@ add_library(crv ${SOURCES})
 # Include directories
 target_include_directories(crv INTERFACE
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 # Link this package to these libraries

doc/CMakeLists.txt

@@ -16,13 +16,13 @@ set(SCOREC_PREFIX "" CACHE STRING "Directory where SCOREC is installed")
 # If SCOREC_PREFIX was specified, only link to that directory,
 # i.e. don't link to another installation in /usr/lib by mistake
 if (SCOREC_PREFIX)
-  find_package(SCOREC 3.0.0 REQUIRED CONFIG PATHS ${SCOREC_PREFIX} NO_DEFAULT_PATH)
+  find_package(SCOREC 4 REQUIRED CONFIG PATHS ${SCOREC_PREFIX} NO_DEFAULT_PATH)
 else()
 # IF SCOREC_PREFIX was not specified, look in typical system directories,
 # and also in CMAKE_PREFIX_PATH (environment variable)
   find_package(
       SCOREC #package name, has to be SCOREC
-      3.0.0  #version. can be omitted, and will match any installed version
+      4      #version. can be omitted, and will match any installed version
              #greater than or equal to this one, as long as the major number
              #is the same
       REQUIRED #indicate that SCOREC is really needed to compile

doc/myprogram.cpp

@@ -1,9 +1,13 @@
 #include <PCU.h>
 #include "mylibrary.h"
 int main(int argc, char** argv) {
+#ifndef SCOREC_NO_MPI
   MPI_Init(&argc,&argv);
-  pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD);
+#endif
+  pcu::PCU PCUObj;
   makeMesh(&PCUObj);
+#ifndef SCOREC_NO_MPI
   MPI_Finalize();
+#endif
   return 0;
 }

dsp/CMakeLists.txt

@@ -28,7 +28,6 @@ add_library(dsp ${SOURCES})
 # Include directories
 target_include_directories(apf INTERFACE
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 # Link this package to these libraries

example/mpi-nompi/CMakeLists.txt

@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.8)
+project(mpi-nompi VERSION 1.0.0 LANGUAGES CXX)
+
+# Allow the user to indicate where they installed SCOREC
+# via "-DSCOREC_PREFIX=/home/somewhere" when calling `cmake`
+set(SCOREC_PREFIX "" CACHE STRING "Directory where SCOREC is installed")
+
+# If SCOREC_PREFIX was specified then use only that directory.
+if (SCOREC_PREFIX)
+  find_package(SCOREC 4 REQUIRED CONFIG PATHS "${SCOREC_PREFIX}" NO_DEFAULT_PATH)
+else()
+  find_package(SCOREC 4 REQUIRED CONFIG)
+endif()
+
+add_executable(hello hello.cc)
+target_link_libraries(hello PRIVATE SCOREC::core)
+add_executable(test_coll test_coll.cc)
+target_link_libraries(test_coll PRIVATE SCOREC::core)
+
+enable_testing()
+add_test(NAME coll1 COMMAND mpirun -np 1 $<TARGET_FILE:test_coll>)
+add_test(NAME coll2 COMMAND mpirun -np 2 $<TARGET_FILE:test_coll>)
+add_test(NAME coll4 COMMAND mpirun -np 4 $<TARGET_FILE:test_coll>)
+

example/mpi-nompi/README.md

@@ -0,0 +1,6 @@
+# Example: Using SCOREC_NO_MPI in an MPI application
+
+This example demonstrates how to combine a SCOREC_NO_MPI build of core with an
+application that is using MPI. This is the prime use case for the SCOREC_NO_MPI
+option.
+

example/mpi-nompi/hello.cc

@@ -0,0 +1,89 @@
+#include <iostream>
+#include <vector>
+
+#include <mpi.h>
+#include <PCU.h>
+#include <gmi_mesh.h>
+#include <apf.h>
+#include <apfBox.h>
+#include <apfMesh2.h>
+
+template <typename T>
+void print_vector(const std::vector<T> &v);
+
+int main(int argc, char** argv) {
+  MPI_Init(&argc, &argv);
+  { // pcu object scope
+  pcu::PCU PCUObj;
+  // Print rank info.
+  int rank, size;
+  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+  MPI_Comm_size(MPI_COMM_WORLD, &size);
+  if (rank == 0)
+    std::cout << "MPI size: " << size << "; PCU size: " <<
+      PCUObj.Peers() << std::endl;
+  for (int i = 0; i < size; ++i) {
+    if (rank == i)
+      std::cout << "Hello from MPI rank: " << rank << "; PCU rank: " <<
+        PCUObj.Self() << std::endl;
+    MPI_Barrier(MPI_COMM_WORLD);
+  }
+  // Test SCOREC functions.
+  gmi_register_mesh();
+  apf::Mesh2* m = apf::makeMdsBox(1, 1, 1, 1, 1, 1, 0, &PCUObj);
+  apf::destroyMesh(m);
+  // MPI Allreduce
+  int val = rank + 1, sum;
+  MPI_Allreduce(&val, &sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+  int pval = rank + 1, psum;
+  psum = PCUObj.Add(pval);
+  for (int i = 0; i < size; ++i) {
+    if (rank == i)
+      std::cout << "MPI(" << rank << ") sum: " << sum <<
+      "; PCU sum: " << psum << std::endl;
+    MPI_Barrier(MPI_COMM_WORLD);
+  }
+  // Test Allgather
+  std::vector<int> vals1(size);
+  MPI_Allgather(&val, 1, MPI_INT, vals1.data(), 1, MPI_INT, MPI_COMM_WORLD);
+  std::vector<int> pvals1(PCUObj.Peers());
+  PCUObj.Allgather(&val, pvals1.data(), 1);
+  for (int i = 0; i < size; ++i) {
+    if (rank == i) {
+      std::cout << "MPI(" << rank << ") gather: ";
+      print_vector(vals1);
+      std::cout << "; PCU gather: ";
+      print_vector(pvals1);
+      std::cout << std::endl;
+    }
+    MPI_Barrier(MPI_COMM_WORLD);
+  }
+  int val2[2] = {val, -val};
+  std::vector<int> vals2(2 * size);
+  MPI_Allgather(val2, 2, MPI_INT, vals2.data(), 2, MPI_INT, MPI_COMM_WORLD);
+  std::vector<int> pvals2(2 * PCUObj.Peers());
+  PCUObj.Allgather(val2, pvals2.data(), 2);
+  for (int i = 0; i < size; ++i) {
+    if (rank == i) {
+      std::cout << "MPI(" << rank << ") gather2: ";
+      print_vector(vals2);
+      std::cout << "; PCU gather2: ";
+      print_vector(pvals2);
+      std::cout << std::endl;
+    }
+    MPI_Barrier(MPI_COMM_WORLD);
+  }
+  } // pcu object scope
+  MPI_Finalize();
+  return 0;
+}
+
+template <typename T>
+void print_vector(const std::vector<T>& v) {
+  std::cout << "[";
+  for (int i = 0; i < v.size(); ++i) {
+    if (i != 0) std::cout << ", ";
+    std::cout << v[i];
+  }
+  std::cout << "]";
+}