Allow MetaPhysicL to run on device using Kokkos

NEWS

@@ -1,3 +1,23 @@
+1.4.0 -> 2.0.0
+  * Improve nvc++ compatibility
+  * Improved parallel testing
+  * Enable and address paranoid warnings
+  * Eliminate code deprecated in c++17
+  * Update autotools
+  * Avoid and test for spurious floating point exceptions
+  * Remove pessimizing std::move
+  * Update autoconf submodule
+  * vexcl.m4 fix when only part of boost is installed
+  * Allow dual numbers to execute on device
+    * Focuses on the SemiDynamicSparseNumberArrayGeneric derivative container
+    * Public do_derivatives static member deprecated
+    * Float and long double overloads removed
+    * Introduction of generic DynamicArrayWrapper
+      * Template using declarations for wrapping std::array and KokkosArray
+      * N template parameter changed from a typename to size_t
+      * typename N parameter moved to SemiDynamicSparseNumberArrayGeneric since
+        base template DynamicSparseNumberBase requires parameter pack of typename
+
 1.3.3 -> 1.4.0
   * Throw exception in opt mode if DynamicStdArrayWrapper size exceeded
   * Add tolerance argument to DynamicSparseNumberBase::sparsity_trim()

configure.ac

@@ -2,7 +2,7 @@
 dnl Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT(metaphysicl, 1.4.0, roystgnr@ices.utexas.edu)
+AC_INIT(metaphysicl, 2.0.0, roystgnr@gmail.com)
 AC_CONFIG_MACRO_DIR([m4])
 
 AC_CONFIG_HEADERS([src/utilities/include/metaphysicl/metaphysicl_config.h.tmp])
@@ -81,6 +81,8 @@ AC_PROG_FC
 # --------------------------------------------------------------
 ACSM_CXX_COMPILER_STANDARD([2011], [2017])
 
+AM_CONDITIONAL(CXX17_ENABLED,test x$HAVE_CXX17 = x1)
+
 AM_CONDITIONAL(CXX14_ENABLED,test x$HAVE_CXX14 = x1)
 
 AM_CONDITIONAL(CXX11_ENABLED,test x$HAVE_CXX11 = x1)
@@ -111,6 +113,29 @@ AC_HAVE_FEEXCEPT
 
 LT_INIT
 
+
+# --------------------------------------------------------------
+# Deprecated code - disable by default
+# --------------------------------------------------------------
+AC_ARG_ENABLE(std-violation,
+              [AS_HELP_STRING([--enable-std-violation],[Add specializations to std::, not MetaPhysicL::])],
+              enable_std_violation=$enableval,
+              enable_std_violation=no)
+
+AC_SUBST(enable_std_violation)
+AS_IF([test "x$enable_std_violation" != xyes],
+      [
+        AC_MSG_RESULT([<<< Configuring library without C++ std:: violations >>>])
+      ],
+      [
+        AC_MSG_RESULT([<<< Configuring library with deprecated C++ std:: violations >>>])
+        AC_DEFINE(ENABLE_STD_VIOLATION, 1, [Flag indicating if the library should violate std::])
+      ])
+# --------------------------------------------------------------
+
+
+
+
 dnl---------------------------------------------------------
 dnl Checks for  library prerequisites for other libraries...
 dnl---------------------------------------------------------
@@ -128,6 +153,9 @@ if (test x$HAVE_VEXCL = x1); then
   antioch_optional_test_LIBS="$VEXCL_LIBS $antioch_optional_test_LIBS"
 fi
 
+dnl Optional check for Kokkos
+CONFIGURE_KOKKOS
+
 dnl build dependencies from right to left, so mpi before timpi
 AS_IF([test "x$enablempi" = xyes],
       [

m4/config_summary.m4

@@ -31,6 +31,10 @@ echo Build host.................... : $BUILD_HOST
 echo Configure date................ : $BUILD_DATE
 echo Build architecture............ : $BUILD_ARCH
 echo Git revision number........... : $BUILD_VERSION
+if test "x$enable_std_violation" = "xyes"; then
+  echo Library settings:
+  echo '  'Enable std:: violations..... : yes
+fi
 echo
 echo Optional Packages for Testing:
 if test "x$HAVE_MASA" = "x1"; then
@@ -60,6 +64,16 @@ if test "x$HAVE_TIMPI" = "x1"; then
 else
   echo '  'TIMPI....................... : no
 fi
+if test "x$KOKKOS_CXX" != "x"; then
+  echo '  'KOKKOS...................... : yes
+  echo '  'KOKKOS_CXX.................. : $KOKKOS_CXX
+  echo '  'KOKKOS_CPPFLAGS............. : $KOKKOS_CPPFLAGS
+  echo '  'KOKKOS_CXXFLAGS............. : $KOKKOS_CXXFLAGS
+  echo '  'KOKKOS_LDFLAGS.............. : $KOKKOS_LDFLAGS
+  echo '  'KOKKOS_LIBS................. : $KOKKOS_LIBS
+else
+  echo '  'KOKKOS...................... : no
+fi
 echo
 echo '-------------------------------------------------------------------------------'
 

m4/kokkos.m4

@@ -0,0 +1,135 @@
+AC_DEFUN([CONFIGURE_KOKKOS],
+[
+  AC_ARG_VAR([KOKKOS_CXXFLAGS], [Extra C++ flags for compiling Kokkos sources])
+
+  dnl -- Backend choice (defaults to CUDA example; pick what you need)
+  AC_ARG_WITH([kokkos-backend],
+    [AS_HELP_STRING([--with-kokkos-backend=BACKEND],
+      [cuda|hip|sycl|openmp|serial (default: cuda)])],
+    [KOKKOS_BACKEND="$withval"], [KOKKOS_BACKEND=cuda])
+
+  dnl -- Kokkos install
+  AC_ARG_WITH([kokkos],
+    [AS_HELP_STRING([--with-kokkos=PATH],
+      [Kokkos prefix (e.g. /opt/kokkos or install prefix)])],
+    [KOKKOS_PREFIX="$withval"], [KOKKOS_PREFIX=""])
+
+  AS_IF([test "x$KOKKOS_PREFIX" != "x"], [
+    KOKKOS_CPPFLAGS="-DMETAPHYSICL_KOKKOS_COMPILATION -I$KOKKOS_PREFIX/include"
+    KOKKOS_CXXFLAGS="--forward-unknown-to-host-compiler $KOKKOS_CXXFLAGS"
+    KOKKOS_LDFLAGS="--forward-unknown-to-host-compiler -L$KOKKOS_PREFIX/lib -Wl,-rpath,$KOKKOS_PREFIX/lib"
+    KOKKOS_LIBS="-lkokkoscore"
+
+    KOKKOS_CFG="$KOKKOS_PREFIX/include/KokkosCore_config.h"
+    AS_IF([! test -r "$KOKKOS_CFG"], [
+      AC_MSG_ERROR([Cannot read $KOKKOS_CFG])
+    ])
+
+    AC_PROG_GREP
+    AC_PROG_SED
+    AS_IF(["$GREP" -F -q '#define KOKKOS_ENABLE_OPENMP' \
+          "$KOKKOS_CFG"],
+      [have_kokkos_openmp=yes],
+      [have_kokkos_openmp=no])
+
+    if test "x$have_kokkos_openmp" = "xyes"; then
+      KOKKOS_CXXFLAGS="-fopenmp $KOKKOS_CXXFLAGS"
+      KOKKOS_LDFLAGS="-fopenmp $KOKKOS_LDFLAGS"
+    fi
+
+    case "$KOKKOS_BACKEND" in
+      cuda)
+        AC_PATH_PROG([NVCC],[nvcc], [no], [$PATH])
+        AS_IF([test "x$NVCC" = "xno"],
+          [AC_MSG_ERROR([nvcc not found. Install CUDA.])])
+        KOKKOS_CXX="$NVCC"
+        KOKKOS_CXXFLAGS="-x cu $KOKKOS_CXXFLAGS"
+
+        dnl
+        dnl credit to ChatGPT for the ensuing parsing of arch's from kokkos config
+        dnl
+
+        dnl harvest defined arch macros from Kokkos config
+        AC_MSG_CHECKING([for Kokkos defined architectures])
+        ax_kokkos_arch_lines=`$GREP '^[[[:space:]]]*#define[[:space:]]\{1,\}KOKKOS_ARCH_[A-Za-z0-9_][A-Za-z0-9_]*' "$KOKKOS_CFG" \
+          | $SED -n 's/.*KOKKOS_ARCH_\([[A-Za-z0-9_]][[A-Za-z0-9_]]*\).*/\1/p'`
+        AC_MSG_RESULT([$ax_kokkos_arch_lines])
+
+        dnl Keep only GPU-ish tokens we know how to map
+        ax_kokkos_arch_gpu=`printf "%s\n" "$ax_kokkos_arch_lines" \
+          | "$GREP" '^\(KEPLER\(30\|32\|35\|37\)\{0,1\}\|MAXWELL\(50\|52\|53\)\{0,1\}\|PASCAL\(60\|61\)\{0,1\}\|VOLTA\(70\|72\)\{0,1\}\|  TURING75\|AMPERE\(80\|86\)\{0,1\}\|ADA89\|HOPPER\(90\)\{0,1\}\|AMD_GFX[0-9A-Za-z]\{1,\}\)$' \
+          || true`
+
+        dnl Prefer numbered macros; if both generic and numbered exist, numbered will appear too.
+        ax_cuda_sms=
+        for t in $ax_kokkos_arch_gpu; do
+          case "$t" in
+            KEPLER30|KEPLER)  ax_cuda_sms="$ax_cuda_sms 30" ;;
+            KEPLER32)         ax_cuda_sms="$ax_cuda_sms 32" ;;
+            KEPLER35)         ax_cuda_sms="$ax_cuda_sms 35" ;;
+            KEPLER37)         ax_cuda_sms="$ax_cuda_sms 37" ;;
+            MAXWELL|MAXWELL50) ax_cuda_sms="$ax_cuda_sms 50" ;;
+            MAXWELL52)        ax_cuda_sms="$ax_cuda_sms 52" ;;
+            MAXWELL53)        ax_cuda_sms="$ax_cuda_sms 53" ;;
+            PASCAL|PASCAL60)  ax_cuda_sms="$ax_cuda_sms 60" ;;
+            PASCAL61)         ax_cuda_sms="$ax_cuda_sms 61" ;;
+            VOLTA|VOLTA70)    ax_cuda_sms="$ax_cuda_sms 70" ;;
+            VOLTA72)          ax_cuda_sms="$ax_cuda_sms 72" ;;
+            TURING75)         ax_cuda_sms="$ax_cuda_sms 75" ;;
+            AMPERE|AMPERE80)  ax_cuda_sms="$ax_cuda_sms 80" ;;
+            AMPERE86)         ax_cuda_sms="$ax_cuda_sms 86" ;;
+            ADA89)            ax_cuda_sms="$ax_cuda_sms 89" ;;
+            HOPPER|HOPPER90)  ax_cuda_sms="$ax_cuda_sms 90" ;;
+            AMD_GFX*)         ;; dnl handled below in HIP section
+          esac
+        done
+
+        dnl Unique & sorted
+        ax_cuda_sms=`printf "%s\n" $ax_cuda_sms | awk '!seen[$0]++'`
+
+        dnl Emit nvcc -gencode flags (compute+code pairs)
+        if test "x$ax_cuda_sms" != x; then
+          for sm in $ax_cuda_sms; do
+            KOKKOS_CXXFLAGS="-gencode=arch=compute_${sm},code=sm_${sm} $KOKKOS_CXXFLAGS"
+          done
+        fi
+        ;;
+      hip)
+        AC_PATH_PROG([HIPCC],[hipcc],[no],[$PATH])
+        AS_IF([test "x$HIPCC" = "xno"],
+          [AC_MSG_ERROR([hipcc not found; install ROCm HIP.])])
+        KOKKOS_CXX="$HIPCC"
+        ;;
+      sycl)
+        AC_PATH_PROG([ICPX],[icpx],[no],[$PATH])
+        AS_IF([test "x$ICPX" = "xno"],
+          [AC_MSG_ERROR([icpx (oneAPI) not found for SYCL backend.])])
+        KOKKOS_CXX="$ICPX"
+        KOKKOS_CXXFLAGS="-fsycl $KOKKOS_CXXFLAGS"
+        ;;
+      openmp)
+        KOKKOS_CXX="$CXX"
+        KOKKOS_CXXFLAGS="-fopenmp $KOKKOS_CXXFLAGS"
+        ;;
+      serial|*)
+        KOKKOS_CXX="$CXX"
+        ;;
+    esac
+
+  ], [
+    KOKKOS_CXX=""
+    KOKKOS_CPPFLAGS=""
+    KOKKOS_CXXFLAGS=""
+    KOKKOS_LDFLAGS=""
+    KOKKOS_LIBS=""
+  ])
+
+
+  AC_SUBST([KOKKOS_CXX])
+  AC_SUBST([KOKKOS_CXXFLAGS])
+  AC_SUBST([KOKKOS_LIBS])
+  AC_SUBST([KOKKOS_CPPFLAGS])
+  AC_SUBST([KOKKOS_LDFLAGS])
+
+  AM_CONDITIONAL([KOKKOS_ENABLED], [test "x$KOKKOS_CXX" != "x"])
+])

src/Makefile.am

@@ -6,7 +6,7 @@ bin_PROGRAMS    = metaphysicl_version
 
 lib_LTLIBRARIES = libmetaphysicl.la
 
-libmetaphysicl_la_LDFLAGS = -version-info 10:0:0
+libmetaphysicl_la_LDFLAGS = -version-info 11:0:0
 
 #----------------------------------------------
 # List of source files to build dynamic library
@@ -28,6 +28,7 @@ include_HEADERS += core/include/metaphysicl/compare_types.h
 include_HEADERS += core/include/metaphysicl/ct_set.h
 include_HEADERS += core/include/metaphysicl/ct_types.h
 include_HEADERS += core/include/metaphysicl/metaphysicl_common.h
+include_HEADERS += core/include/metaphysicl/metaphysicl_device.h
 
 # graphs
 include_HEADERS += graphs/include/metaphysicl/physics.h
@@ -68,6 +69,7 @@ include_HEADERS += numerics/include/metaphysicl/numberarray.h
 include_HEADERS += numerics/include/metaphysicl/numbervector.h
 include_HEADERS += numerics/include/metaphysicl/raw_type.h
 include_HEADERS += numerics/include/metaphysicl/shadownumber.h
+include_HEADERS += numerics/include/metaphysicl/shadownumber_decl.h
 include_HEADERS += numerics/include/metaphysicl/sparsenumberarray.h
 include_HEADERS += numerics/include/metaphysicl/sparsenumberstruct.h
 include_HEADERS += numerics/include/metaphysicl/sparsenumberutils.h
@@ -88,9 +90,13 @@ include_HEADERS += utilities/include/metaphysicl/metaphysicl_exceptions.h
 include_HEADERS += utilities/include/metaphysicl/metaprogramming.h
 include_HEADERS += utilities/include/metaphysicl/testable.h
 include_HEADERS += utilities/include/metaphysicl/dynamic_std_array_wrapper.h
-include_HEADERS += utilities/include/metaphysicl/parallel_dynamic_std_array_wrapper.h
+include_HEADERS += utilities/include/metaphysicl/dynamic_kokkos_array_wrapper.h
+include_HEADERS += utilities/include/metaphysicl/dynamic_array_wrapper.h
+include_HEADERS += utilities/include/metaphysicl/parallel_dynamic_array_wrapper.h
 include_HEADERS += utilities/include/metaphysicl/ignore_warnings.h
 include_HEADERS += utilities/include/metaphysicl/restore_warnings.h
+include_HEADERS += utilities/include/metaphysicl/metaphysicl_math.h
+include_HEADERS += utilities/include/metaphysicl/metaphysicl_numeric_limits.h
 
 # Needs to be builddir since this is generated by configure
 include_HEADERS += $(top_builddir)/src/utilities/include/metaphysicl/metaphysicl_version.h

src/core/include/metaphysicl/metaphysicl_device.h

@@ -0,0 +1,41 @@
+#ifndef METAPHYSICL_DEVICE_H
+#define METAPHYSICL_DEVICE_H
+
+#ifdef METAPHYSICL_KOKKOS_COMPILATION
+
+#include <Kokkos_Array.hpp>
+#include <Kokkos_Macros.hpp>
+
+#define METAPHYSICL_INLINE KOKKOS_INLINE_FUNCTION
+#define METAPHYSICL_IF_ON_HOST(...)                                            \
+  do {                                                                         \
+    KOKKOS_IF_ON_HOST(__VA_ARGS__)                                             \
+  } while (0)
+
+namespace MetaPhysicL {
+template <typename T> METAPHYSICL_INLINE auto begin(T &t) {
+  return Kokkos::begin(t);
+}
+} // namespace MetaPhysicL
+
+#else
+
+#include <array>
+
+#define METAPHYSICL_INLINE inline
+// Helper to remove one level of parentheses
+#define METAPHYSICL_UNPAREN(...) __VA_ARGS__
+
+// Make it a standalone statement (no trailing ';' required), and
+// unparen so ( ... ; ) becomes ... ;
+#define METAPHYSICL_IF_ON_HOST(CODE)                                           \
+  do {                                                                         \
+    METAPHYSICL_UNPAREN CODE                                                   \
+  } while (0)
+
+namespace MetaPhysicL {
+template <typename T> auto begin(T &t) { return std::begin(t); }
+} // namespace MetaPhysicL
+#endif
+
+#endif // METAPHYSICL_DEVICE_H

src/numerics/include/metaphysicl/dualderivatives.h

@@ -56,15 +56,15 @@ struct DivergenceType
 
 
 template <typename T>
-inline
+METAPHYSICL_INLINE
 typename DerivativeType<T>::type
 derivative(const T& a, unsigned int derivativeindex)
 {
   return a.derivatives()[derivativeindex];
 }
 
 template <typename T>
-inline
+METAPHYSICL_INLINE
 typename DerivativesType<T>::type
 derivatives(const T& a)
 {