Load balancing across GPUs

.github/CI/github_runner.yaml

@@ -1,7 +1,7 @@
 spack:
   definitions:
   - pkgs:
-    - gcc@11.2.0
+    - gcc@12.1.0
     - git
     - patch
     - flex
@@ -11,7 +11,7 @@ spack:
     - py-cython
     - cmake
     - ninja
-    - otf2
+    - otf2@2.3
     - openmpi
 
   view: true

.github/CI/spack_setup.sh

@@ -28,14 +28,19 @@ save_dir=`pwd`
 #
 echo "::group::Spack environment setup"
 if [ -r ${SPACK_DIR}/.git/FETCH_HEAD ]; then
+  # We should never allow spack to update it's packages, it increases the
+  # opportunity for mishandling of the runner environment. Instead, when
+  # there is a need for update, the entire runner should be flushed, such
+  # that a fresh spack install is created.
   last_update=`stat -c %Y ${SPACK_DIR}/.git/FETCH_HEAD`
   current=`date -d now "+%s"`
-  if [ $(((current - last_update) / 86400)) -gt 1 ]; then
-    echo "Last update ${last_update}, current ${current}: Do git pull spack"
-    cd $SPACK_DIR && git pull
-  else
-    echo "git pull was less than one day ago"
-  fi
+  echo "spack is $(((current - last_update) / 86400)) days old"
+  # if [ $(((current - last_update) / 86400)) -gt 1 ]; then
+  #   echo "Last update ${last_update}, current ${current}: Do git pull spack"
+  #   cd $SPACK_DIR && git pull
+  # else
+  #   echo "git pull was less than one day ago"
+  # fi
 else
   echo "git clone spack"
   git clone https://github.com/spack/spack $SPACK_DIR || true
@@ -44,14 +49,15 @@ else
   cd $SPACK_DIR && git pull && git status
 fi
 
+rm -rf ${HOME}/.spack
+
 echo "Load spack environment"
 source $SPACK_DIR/share/spack/setup-env.sh
 
-spack external find
 spack compiler find
 
 # Start with a fresh env every time
-spack env remove -y ${RUNNER_ENV}
+spack env remove -y ${RUNNER_ENV} || true
 
 cd ${save_dir}
 mkdir ${RUNNER_ENV} || true

.github/workflows/build_cmake.yml

@@ -60,7 +60,8 @@ jobs:
       # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
       run: |
         source ${{github.workspace}}/.github/CI/spack_setup.sh
-        cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE $BUILD_CONFIG CC=gcc CXX=g++
+        echo CC=gcc CXX=g++ FC=gfortran cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE $BUILD_CONFIG
+        CC=gcc CXX=g++ FC=gfortran cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE $BUILD_CONFIG
 
     - name: Build
       working-directory: ${{ env.BUILD_DIRECTORY }}

cmake_modules/FindOTF2.cmake

@@ -63,9 +63,9 @@ ELSE(NOT OTF2_CONFIG)
             ${OTF2_LINK_DIRS}
             )
           IF(_OTF2_LIB_FROM_ARG)
-	    IF(_OTF2_LIBRARY_TEST)
-	      SET(OTF2_LIBRARY ${_OTF2_LIB_FROM_ARG})
-	    ENDIF(_OTF2_LIBRARY_TEST)
+            IF(_OTF2_LIBRARY_TEST)
+              SET(OTF2_LIBRARY ${_OTF2_LIB_FROM_ARG})
+            ENDIF(_OTF2_LIBRARY_TEST)
             SET(OTF2_LIBRARIES ${OTF2_LIBRARIES} ${_OTF2_LIB_FROM_ARG})
           ENDIF(_OTF2_LIB_FROM_ARG)
           UNSET(_OTF2_LIB_FROM_ARG CACHE)

cmake_modules/PaRSECConfig.cmake.in

@@ -20,8 +20,10 @@ find_package(Threads)
 
 if(@PARSEC_HAVE_HWLOC@)
   set_and_check(HWLOC_INCLUDE_DIR "@HWLOC_INCLUDE_DIR@")
+  set(HWLOC_INCLUDE_DIR ${HWLOC_INCLUDE_DIR} CACHE PATH "Imported by PaRSECConfig.cmake" FORCE)
   set_and_check(HWLOC_LIBRARY "@HWLOC_LIBRARY@")
-  find_package(HWLOC REQUIRED)
+  set(HWLOC_LIBRARY ${HWLOC_LIBRARY} CACHE PATH "Imported by PaRSECConfig.cmake" FORCE)
+  find_package(HWLOC REQUIRED MODULE)
 endif(@PARSEC_HAVE_HWLOC@)
 
 if(@PARSEC_HAVE_OTF2@)
@@ -40,8 +42,10 @@ endif(@PARSEC_HAVE_OTF2@)
 
 if(@PARSEC_HAVE_PAPI@)
   set_and_check(PAPI_INCLUDE_DIR "@PAPI_INCLUDE_DIR@")
+  set(PAPI_INCLUDE_DIR ${PAPI_INCLUDE_DIR} CACHE PATH "Imported by PaRSECConfig.cmake" FORCE)
   set_and_check(PAPI_LIBRARY "@PAPI_LIBRARY@")
-  find_package(PAPI REQUIRED)
+  set(PAPI_LIBRARY ${PAPI_LIBRARY} CACHE PATH "Imported by PaRSECConfig.cmake" FORCE)
+  find_package(PAPI REQUIRED MODULE)
 endif(@PARSEC_HAVE_PAPI@)
 
 if(@PARSEC_DIST_WITH_MPI@)

cmake_modules/pregen_flex_bison.cmake

@@ -49,6 +49,7 @@ if(archive)
     # Check for problems: stray files in build directory
     execute_process(
       COMMAND git ls-files --error-unmatch ${srcdir}/${source}
+      WORKING_DIRECTORY ${srcdir}
       RESULT_VARIABLE ret
       OUTPUT_VARIABLE stdout
       ERROR_VARIABLE stderr
@@ -61,6 +62,7 @@ if(archive)
     # Check for modifications
     execute_process(
       COMMAND git status --porcelain -- ${srcdir}/${source}
+      WORKING_DIRECTORY ${srcdir}
       RESULT_VARIABLE ret
       OUTPUT_VARIABLE stdout
       ERROR_VARIABLE stderr

parsec/CMakeLists.txt

@@ -42,10 +42,12 @@ set(BASE_SOURCES
 
 if(FLEX_FOUND AND BISON_FOUND)
   # generate in the build dir
+  file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/utils")
   FLEX_TARGET(keyval_flex utils/keyval_lex.l utils/keyval_lex.l.c)
   FLEX_TARGET(show_help_flex utils/show_help_lex.l utils/show_help_lex.l.c)
 
   # Generate in the pregen dir, add to the non-automated rule
+  file(MAKE_DIRECTORY "${PARSEC_PREGEN_FLEX_BISON_DIR}/parsec/utils")
   FLEX_TARGET(pregen_keyval_flex utils/keyval_lex.l ${PARSEC_PREGEN_FLEX_BISON_DIR}/parsec/utils/keyval_lex.l.c COMPILE_FLAGS --noline)
   FLEX_TARGET(pregen_show_help_flex utils/show_help_lex.l ${PARSEC_PREGEN_FLEX_BISON_DIR}/parsec/utils/show_help_lex.l.c COMPILE_FLAGS --noline)
   add_custom_target(parsec_pregen_flex_utils SOURCES ${FLEX_pregen_keyval_flex_OUTPUTS} ${FLEX_pregen_show_help_flex_OUTPUTS})
@@ -111,6 +113,9 @@ set(SOURCES
   mempool.c
   private_mempool.c
   remote_dep.c
+  parsec_comm_engine.c
+  parsec_mpi_funnelled.c
+  remote_dep_mpi.c
   scheduling.c
   compound.c
   vpmap.c
@@ -229,6 +234,7 @@ if( BUILD_PARSEC )
     $<$<BOOL:${PARSEC_HAVE_OTF2}>:OTF2::OTF2>
     $<$<BOOL:${MPI_C_FOUND}>:MPI::MPI_C>
     $<$<BOOL:${PARSEC_HAVE_CUDA}>:CUDA::cudart>
+    $<$<BOOL:${PARSEC_HAVE_CUDA}>:CUDA::nvml>
     ${EXTRA_LIBS}
     INTERFACE
     ${PARSEC_ATOMIC_SUPPORT_LIBS}
@@ -266,11 +272,17 @@ if( BUILD_PARSEC )
   get_target_property(_public_headers parsec PUBLIC_HEADER_H)
   foreach(_FILE ${_public_headers})
     get_filename_component(_DIR ${_FILE} DIRECTORY)
+    if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${_FILE}")
+      set(_FILE "${CMAKE_CURRENT_BINARY_DIR}/${_FILE}")
+    endif()
     INSTALL(FILES ${_FILE} DESTINATION ${PARSEC_INSTALL_INCLUDEDIR}/parsec/${_DIR})
   endforeach()
   get_target_property(_private_headers parsec PRIVATE_HEADER_H)
   foreach(_FILE ${_private_headers})
     get_filename_component(_DIR ${_FILE} DIRECTORY)
+    if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${_FILE}")
+      set(_FILE "${CMAKE_CURRENT_BINARY_DIR}/${_FILE}")
+    endif()
     INSTALL(FILES ${_FILE} DESTINATION ${PARSEC_INSTALL_INCLUDEDIR}/parsec/${_DIR})
   endforeach()
 
@@ -287,6 +299,8 @@ if( BUILD_PARSEC )
           ${CMAKE_CURRENT_SOURCE_DIR}/dictionary.h
           ${CMAKE_CURRENT_SOURCE_DIR}/data.h
           ${CMAKE_CURRENT_SOURCE_DIR}/private_mempool.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/bindthread.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/parsec_comm_engine.h
           ${CMAKE_CURRENT_SOURCE_DIR}/include/parsec/deprecated.h
           DESTINATION ${PARSEC_INSTALL_INCLUDEDIR}/parsec)