Merge branch 'master' into kmp5/feature/CP

Author: kmp5VT

.github/workflows/ci.yml

@@ -12,16 +12,26 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os : [ macos-latest ]
-        cxx : [ clang++, /usr/local/bin/g++-10 ]
+        os : [ macos-latest, ubuntu-22.04 ]
         build_type : [ Release, Debug ]
         task_backend: [ Pthreads, PaRSEC ]
-        prerequisites : [ gcc@10 boost eigen open-mpi bison scalapack ]
+        include:
+          - os: ubuntu-22.04
+            cc: /usr/bin/gcc-12
+            cxx: /usr/bin/g++-12
+          - os: macos-latest
+            cc: clang
+            cxx: clang++
 
     name: "${{ matrix.os }}: ${{ matrix.cxx }} ${{ matrix.build_type }} ${{ matrix.task_backend }}"
     runs-on: ${{ matrix.os }}
     env:
       CXX : ${{ matrix.cxx }}
+      CCACHE_DIR : ${{github.workspace}}/build/.ccache
+      CCACHE_COMPRESS : true
+      CCACHE_COMPRESSLEVEL : 6
+      OMPI_MCA_btl_vader_single_copy_mechanism : none
+      PARSEC_MCA_runtime_bind_threads : 0
       BUILD_CONFIG : >
         -DMADNESS_TASK_BACKEND=${{ matrix.task_backend }}
         -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
@@ -33,37 +43,48 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - uses: maxim-lobanov/setup-xcode@v1
-      with:
-        xcode-version: '<14'
-
     - name: Host system info
       shell: bash
       run: cmake -P ${{github.workspace}}/ci/host_system_info.cmake
 
-    - name: Install ${{matrix.prerequisites}}
+
+    - name: Install prerequisite MacOS packages
+      if: ${{ matrix.os == 'macos-latest' }}
       run: |
-        brew install ${{matrix.prerequisites}}
-        echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
+        brew install ninja boost eigen open-mpi bison scalapack ccache
+        echo "MPIEXEC=/opt/homebrew/bin/mpiexec" >> $GITHUB_ENV
+
+    - name: Install prerequisites Ubuntu packages
+      if: ${{ matrix.os == 'ubuntu-22.04' }}
+      run: |
+        wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
+        sudo apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main"
+        sudo apt-get update
+        sudo apt-get -y install ninja-build g++-12 liblapack-dev libboost-dev libboost-serialization-dev libboost-random-dev libeigen3-dev openmpi-bin libopenmpi-dev libtbb-dev ccache flex bison libscalapack-mpi-dev cmake doxygen
+        sudo ln -s /usr/lib/x86_64-linux-gnu/libscalapack-openmpi.so /usr/lib/x86_64-linux-gnu/libscalapack.so
+        echo "MPIEXEC=/usr/bin/mpiexec" >> $GITHUB_ENV
+
+    - name: Setup ccache
+      uses: hendrikmuhs/ccache-action@v1.2
+      with:
+        key: ccache-${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.task_backend }}
 
     - name: "Configure build: ${{ env.BUILD_CONFIG }}"
       shell: bash
       run: |
         set -x;
-        cmake -B${{github.workspace}}/build $BUILD_CONFIG || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log)
+        cmake -B${{github.workspace}}/build $BUILD_CONFIG || (cat CMakeFiles/CMakeConfigureLog.yaml)
 
     - name: Build
       working-directory: ${{github.workspace}}/build
       shell: bash
       run: |
-        cmake --build . --target tiledarray
-        cmake --build . --target examples
+        ccache -p && ccache -z && cmake --build . --target tiledarray ta_test examples && ccache -s
 
     - name: Test
       working-directory: ${{github.workspace}}/build
       shell: bash
       #run: ctest -C $${{matrix.build_type}}
       run: |
         source ${{github.workspace}}/ci/openmpi.env
-        cmake --build . --target ta_test
         cmake --build . --target check-tiledarray

.gitlab-ci.yml

@@ -15,6 +15,7 @@ variables:
     TA_UT_CTEST_TIMEOUT=3000
     ${TA_PYTHON}
     ${ENABLE_CUDA}
+    CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
     ${BLA_VENDOR}
     ${BLA_THREADS}
     ${ENABLE_SCALAPACK}
@@ -24,11 +25,18 @@ before_script:
   # TODO optimize ta_test build memory consumption
   - export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:=1}
   - echo "CMAKE_BUILD_PARALLEL_LEVEL=$CMAKE_BUILD_PARALLEL_LEVEL"
+  # configure ccache
+  - export CCACHE_DIR=/root/.ccache
+  - export CCACHE_COMPRESS=true
+  - export CCACHE_COMPRESSLEVEL=6
+  # print out the ccache configuration
+  - ccache -p
+  # zero out the ccache statistics
+  - ccache -z
 
 ubuntu:
   stage: build
   tags:
-    - docker
     - ${RUNNER_TAGS}
   timeout: 3h
   image: valeevgroup/${IMAGE}
@@ -52,26 +60,20 @@ ubuntu:
     - build/CMakeCache.txt
     - build/CMakeFiles/CMakeOutput.log
     - build/CMakeFiles/CMakeError.log
+    - build/CMakeFiles/CMakeConfigureLog.yaml
     reports:
       metrics: build/metrics.txt
   parallel:
     matrix:
-      - IMAGE : [ "ubuntu:18.04", "ubuntu:20.04" ]
+      - IMAGE : [ "ubuntu:22.04" ]
         CXX: [ g++ ]
-        BUILD_TYPE : [ "Release" ]
-        BLA_VENDOR : [ "BLAS_PREFERENCE_LIST=IntelMKL" ]
-        BLA_THREADS : [ "IntelMKL_THREAD_LAYER=tbb" ]
-        # ENABLE_SCALAPACK : [ "ENABLE_SCALAPACK=ON", "ENABLE_SCALAPACK=OFF" ]
-        TA_PYTHON : [ "TA_PYTHON=OFF" ] # needs to be fixed for MKL
-        RUNNER_TAGS: [ linux ]
-      - IMAGE : [ "ubuntu:18.04", "ubuntu:20.04" ]
-        CXX: [ g++, clang++-9 ]
-        BUILD_TYPE : [ "Release", "Debug" ]
-        ENABLE_SCALAPACK : [ "ENABLE_SCALAPACK=ON", "ENABLE_SCALAPACK=OFF" ]
-        RUNNER_TAGS: [ linux ]
-      - IMAGE : [ "ubuntu:18.04", "ubuntu:20.04" ]
-        CXX: [ g++ ]
-        BUILD_TYPE : [ "Release", "Debug" ]
+        BUILD_TYPE : [ "RelWithDebInfo" ]
+        TA_PYTHON : [ "TA_PYTHON=OFF" ]
         ENABLE_CUDA : [ "ENABLE_CUDA=ON" ]
         TA_TARGETS : [ "tiledarray examples-tiledarray check_serial-tiledarray" ]
         RUNNER_TAGS: [ cuda ]
+
+
+after_script:
+  # print out the ccache statistics
+  - ccache -s

.pre-commit-config.yaml

@@ -38,5 +38,5 @@ repos:
                 name: Format C/C++ code using clang-format.
                 language: system
                 files: \.(c|cc|cxx|cpp|h|hpp|hxx)$
-                entry: clang-format -i
-                args: [--style=file]
+                entry: bin/admin/clang-format.sh
+                args: [--style=file -i]

CMakeLists.txt

@@ -78,25 +78,23 @@ enable_language(C)  # C needed even for basic platform introspection
 
 # Set install paths ============================================================
 
-set(TILEDARRAY_INSTALL_BINDIR "bin"
-    CACHE PATH "TiledArray binary install directory")
-set(TILEDARRAY_INSTALL_INCLUDEDIR "include"
-    CACHE PATH "TiledArray INCLUDE install directory")
-set(TILEDARRAY_INSTALL_LIBDIR "lib"
-    CACHE PATH "TiledArray LIB install directory")
-set(TILEDARRAY_INSTALL_SHAREDIR "share/tiledarray/${TILEDARRAY_EXT_VERSION}"
-    CACHE PATH "TiledArray DATA install directory")
-set(TILEDARRAY_INSTALL_DATADIR "${TILEDARRAY_INSTALL_SHAREDIR}/data"
-    CACHE PATH "TiledArray DATA install directory")
-set(TILEDARRAY_INSTALL_DOCDIR "${TILEDARRAY_INSTALL_SHAREDIR}/doc"
-    CACHE PATH "TiledArray DOC install directory")
-set(TILEDARRAY_INSTALL_CMAKEDIR "lib/cmake/tiledarray"
-    CACHE PATH "TiledArray CMAKE install directory")
+include(GNUInstallDirs)
+set(TILEDARRAY_INSTALL_BINDIR "${CMAKE_INSTALL_BINDIR}"
+        CACHE PATH "TiledArray binary install directory")
+set(TILEDARRAY_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}"
+        CACHE PATH "TiledArray INCLUDE install directory")
+set(TILEDARRAY_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}"
+        CACHE PATH "TiledArray LIB install directory")
+set(TILEDARRAY_INSTALL_DATADIR "${CMAKE_INSTALL_DATAROOTDIR}/tiledarray/${TILEDARRAY_EXT_VERSION}"
+        CACHE PATH "TiledArray DATA install directory")
+set(TILEDARRAY_INSTALL_DOCDIR "${TILEDARRAY_INSTALL_DATADIR}/doc"
+        CACHE PATH "TiledArray DOC install directory")
+set(TILEDARRAY_INSTALL_CMAKEDIR "${CMAKE_INSTALL_LIBDIR}/cmake/tiledarray"
+        CACHE PATH "TiledArray CMAKE install directory")
 
 # Add module directory and modules =============================================
 list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/modules/)
 include(CMakePushCheckState)
-include(GNUInstallDirs)
 include(AppendFlags)
 include(RedefaultableOption)
 include(DetectMADNESSConfig)
@@ -111,6 +109,7 @@ include(LoadFetchContent)
 include(CMakeDependentOption)
 include(CMakePackageConfigHelpers)
 include(FeatureSummary)
+include(CTest)  # testing, defined BUILD_TESTING
 
 set(MPI_CXX_SKIP_MPICXX TRUE CACHE BOOL "MPI_CXX_SKIP_MPICXX")
 
@@ -130,10 +129,8 @@ add_feature_info(TBB ENABLE_TBB "Intel Thread-Building Blocks (TBB) supports pro
 option(ENABLE_CUDA "Enable use of CUDA with TiledArray" OFF)
 add_feature_info(CUDA ENABLE_CUDA "NVIDIA CUDA support for GPU")
 
-if(ENABLE_CUDA)
-  option(ENABLE_CUDA_ERROR_CHECK "TiledArray will always check errors in CUDA calls" ON)
-  add_feature_info(CUDA_ERROR_CHECK ENABLE_CUDA_ERROR_CHECK "Checks CUDA Error")
-endif()
+option(ENABLE_HIP "Enable use of HIP with TiledArray" OFF)
+add_feature_info(HIP ENABLE_HIP "AMD HIP/ROCm support for GPU")
 
 option(ENABLE_GPERFTOOLS "Enable linking with Gperftools" OFF)
 add_feature_info(GPERFTOOLS ENABLE_GPERFTOOLS "Google Performance Tools provide fast memory allocation and performance profiling")
@@ -168,12 +165,16 @@ if(TA_ENABLE_TILE_OPS_LOGGING AND NOT DEFINED TA_TILE_OPS_LOG_LEVEL)
   set(TA_TILE_OPS_LOG_LEVEL 1)
 endif(TA_ENABLE_TILE_OPS_LOGGING AND NOT DEFINED TA_TILE_OPS_LOG_LEVEL)
 
-option(TA_RANGEV3 "Enable Range-V3 library" OFF)
-add_feature_info(TA_RANGEV3 TA_RANGEV3 "Range-V3 ranges library")
+option(TA_TRACE_GLOBAL_COMM_STATS "Enable tracing of communication stats of global objects (DistEval's and DIstributedStorage) TiledArray" OFF)
+add_feature_info(TASK_TRACE_DEBUG TA_TRACE_GLOBAL_COMM_STATS "Debug communication stats of global objects (DistEval's and DIstributedStorage) TiledArray")
+set(TILEDARRAY_ENABLE_GLOBAL_COMM_STATS_TRACE ${TA_TRACE_GLOBAL_COMM_STATS})
 
 option(TA_TTG "Enable search/build of TTG library" OFF)
 add_feature_info(TA_TTG TA_TTG "TTG library")
 
+option(IntelMKL_FAIR_DISPATCH "Enable fair dispatch in Intel MKL" OFF)
+add_feature_info(IntelMKL_FAIR_DISPATCH IntelMKL_FAIR_DISPATCH "Use of fair dispatch in Intel MKL")
+
 # Enable shared library support options
 redefaultable_option(TA_ASSUMES_ASLR_DISABLED "TiledArray assumes the Address Space Layout Randomization (ASLR) to be disabled" OFF)
 add_feature_info(ASSUMES_ASLR_DISABLED TA_ASSUMES_ASLR_DISABLED
@@ -288,13 +289,6 @@ set_property(
   CACHE TA_ASSERT_POLICY PROPERTY
   STRINGS TA_ASSERT_THROW TA_ASSERT_ABORT TA_ASSERT_IGNORE)
 
-# if building unit tests default to throw to be able to test TA_ASSERT statements
-if (NOT (TA_ASSERT_POLICY STREQUAL TA_ASSERT_THROW))
-  if (BUILD_TESTING)
-    message(FATAL_ERROR "TA_ASSERT_POLICY=${TA_ASSERT_POLICY} requires BUILD_TESTING=OFF")
-  endif(BUILD_TESTING)
-endif()
-
 ##########################
 # Include source dirctories
 ##########################
@@ -305,15 +299,29 @@ include_directories(${PROJECT_SOURCE_DIR}/src ${PROJECT_BINARY_DIR}/src)
 ##########################
 add_custom_target(External-tiledarray)
 
+# ccache is an optional dep but must be found first so that the rest of dependencies can use it
+find_program(CCACHE ccache)
+if(CCACHE)
+  mark_as_advanced(CCACHE)
+  message (STATUS "Found ccache: ${CCACHE}")
+  set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE}" CACHE STRING "Compiler launcher to use for compiling C++")
+  set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE}" CACHE STRING "Compiler launcher to use for compiling C")
+  set(CMAKE_CUDA_COMPILER_LAUNCHER "${CCACHE}" CACHE STRING "Compiler launcher to use for compiling CUDA")
+endif(CCACHE)
+
 # required deps:
-# 1. CUDA first since others may depend on it
+# 1. derive runtime (CUDA/HIP/...) first since others may depend on it
 if(ENABLE_CUDA)
   include(external/cuda.cmake)
 endif()
+if(ENABLE_HIP)
+  include(external/hip.cmake)
+endif()
+include(${PROJECT_SOURCE_DIR}/cmake/modules/FindOrFetchRangeV3.cmake)
+include(${PROJECT_SOURCE_DIR}/cmake/modules/FindOrFetchMADWorld.cmake)
 if (TA_TTG)
   include(${PROJECT_SOURCE_DIR}/cmake/modules/FindOrFetchTTG.cmake)
 endif(TA_TTG)
-include(${PROJECT_SOURCE_DIR}/cmake/modules/FindOrFetchMADWorld.cmake)
 detect_MADNESS_configuration()
 include(external/eigen.cmake)
 # the FetchContent-based version will not work due to BLT target name conflicts
@@ -331,28 +339,15 @@ if (ENABLE_WFN91_LINALG_DISCOVERY_KIT)
   include(FetchWfn91LinAlgModules)
   include(FindLinalg)
 endif(ENABLE_WFN91_LINALG_DISCOVERY_KIT)
-# BTAS does a better job of building and checking Boost since it uses Boost::serialization
-# it also memorized the location of its config for use from install tree
+# Boost is to be discovered by the top cmake project, and every (sub)project needs to make sure it has all of its targets
+include(external/boost.cmake)
 include(${PROJECT_SOURCE_DIR}/cmake/modules/FindOrFetchBTAS.cmake)
-include(${PROJECT_SOURCE_DIR}/cmake/modules/FindOrFetchBoost.cmake)
 if(ENABLE_SCALAPACK)
   include(external/scalapackpp.cmake)
 endif()
 
-# optional deps:
-# 1. ccache
-find_program(CCACHE ccache)
-if(CCACHE)
-    mark_as_advanced(CCACHE)
-    message (STATUS "Found ccache: ${CCACHE}")
-    set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE}" CACHE STRING "Compiler launcher to use for compiling C++")
-    set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE}" CACHE STRING "Compiler launcher to use for compiling C")
-endif(CCACHE)
-# 2. range-v3
-if (TA_RANGEV3)
-  include(${PROJECT_SOURCE_DIR}/cmake/modules/FindOrFetchRangeV3.cmake)
-endif(TA_RANGEV3)
-# 3. TTG
+# other optional deps:
+# 2. TTG
 # N.B. make sure TA configures MADNESS correctly
 #if (TA_TTG)
 #  include(${PROJECT_SOURCE_DIR}/cmake/modules/FindOrFetchTTG.cmake)
@@ -369,14 +364,20 @@ add_subdirectory(doc)
 ##########################
 # checking/testing
 ##########################
-include(CTest)
+# N.B. CTest was included above
 if (BUILD_TESTING)
   set(_ctest_args -V -R "tiledarray/unit/run-np.*")
   set(_ctest_args_serial -V -R "tiledarray/unit/run-np-1")
   if (DEFINED TA_UT_CTEST_TIMEOUT)
     list(APPEND _ctest_args --timeout ${TA_UT_CTEST_TIMEOUT})
     list(APPEND _ctest_args_serial --timeout ${TA_UT_CTEST_TIMEOUT})
   endif(DEFINED TA_UT_CTEST_TIMEOUT)
+
+  # if building unit tests need to configure with TA_ASSERT_POLICY=TA_ASSERT_THROW to be able to test TA_ASSERT statements
+  if (NOT (TA_ASSERT_POLICY STREQUAL TA_ASSERT_THROW))
+    message(WARNING "BUILD_TESTING=ON requires configuring with TA_ASSERT_POLICY=TA_ASSERT_THROW to engage REQUIRE_THROWS() tests; will skip these tests")
+  endif()
+
   add_custom_target_subproject(tiledarray check USES_TERMINAL COMMAND ${CMAKE_CTEST_COMMAND} ${_ctest_args})
   add_custom_target_subproject(tiledarray check_serial USES_TERMINAL COMMAND ${CMAKE_CTEST_COMMAND} ${_ctest_args_serial})
   add_subdirectory(tests)
@@ -431,7 +432,7 @@ CONFIGURE_FILE(
 
 # install config files
 install(FILES ${PROJECT_BINARY_DIR}/tiledarray.pc
-    DESTINATION lib/pkgconfig)
+    DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 
 # include extra cmake files
 install(FILES
@@ -480,8 +481,11 @@ ADD_CUSTOM_TARGET(release
   COMMENT "Switch CMAKE_BUILD_TYPE to Release"
   )
 
-feature_summary(WHAT ALL
-                DESCRIPTION "=== TiledArray Package/Feature Info ===")
+if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
+  feature_summary(WHAT ALL
+                  DESCRIPTION "=== TiledArray Package/Feature Info ===")
+  feature_summary(FILENAME ${CMAKE_CURRENT_BINARY_DIR}/features.log WHAT ALL)
+endif()
 
 option(TA_PYTHON "Build TA python module" OFF)
 if (TA_PYTHON)