Changes necessary to integrate AMS into a large multiphysics code

CMakeLists.txt

@@ -21,6 +21,7 @@ endif()
 set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH};")
 
 set(AMS_APP_LIBRARIES "")
+set(AMS_APP_PRIVATE_LIBRARIES "")
 set(AMS_APP_DEFINES "")
 set(AMS_APP_INCLUDES "")
 
@@ -39,13 +40,16 @@ option(WITH_TORCH_DEBUG    "Compute RMSE of Surrogate Model and Physics Module"
 option(WITH_TESTS          "Compile tests" OFF)
 option(WITH_REDIS          "Use REDIS as a database back end" OFF)
 option(WITH_HDF5           "Use HDF5 as a database back end" OFF)
+option(HDF5_USE_STATIC_LIBRARIES "Use static HDF5." OFF)
+set(HDF5_WITH_ZLIB "" CACHE FILETYPE "Use the following zlib for HDF5")
 option(WITH_RMQ            "Use RabbitMQ as a database back end (require a reachable and running RabbitMQ server service)" OFF)
 option(WITH_AMS_DEBUG      "Enable verbose messages" OFF)
 option(WITH_PERFFLOWASPECT "Use PerfFlowAspect for Profiling" OFF)
 option(WITH_WORKFLOW       "Install python drivers used by the outer workflow" OFF)
 option(WITH_AMS_LIB        "Install C++ library to support scientific applications" ON)
 option(WITH_ADIAK          "Use Adiak for recording metadata" OFF)
 option(BUILD_SHARED_LIBS   "Build using shared libraries" ON)
+option(EXCLUDE_STATIC_LIBS "Exclude static libs from the linking line" OFF)
 
 if (WITH_MPI)
   #  SET(CMAKE_CXX_COMPILER "${MPI_CXX_COMPILER}" CACHE FILEPATH "CXX compiler overridden with MPI C++ wrapper")
@@ -56,7 +60,6 @@ if (WITH_MPI)
   message(STATUS "MPICXX: ${MPI_CXX_COMPILER}")
   list(APPEND AMS_APP_LIBRARIES MPI::MPI_CXX)
   message(STATUS "MPI Library used: " MPI::MPI_CXX)
-  list(APPEND AMS_APP_DEFINES "-D__ENABLE_MPI__")
 endif()
 
 # ------------------------------------------------------------------------------
@@ -69,6 +72,7 @@ if (WITH_CUDA)
   if (BUILD_SHARED_LIBS)
     set(CUDA_RUNTIME_LIBRARY "Shared")
   else()
+    set(HDF5_USE_STATIC_LIBRARIES ON)
     set(CUDA_RUNTIME_LIBRARY "Static")
   endif()
 
@@ -132,14 +136,17 @@ endif() # WITH_REDIS
 
 if (WITH_HDF5)
   if (HDF5_USE_STATIC_LIBRARIES)
-  find_package(HDF5 NAMES hdf5 COMPONENTS C static NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake)
-  list(APPEND AMS_APP_LIBRARIES ${HDF5_C_STATIC_LIBRARY})
-  message(STATUS "HDF5 Static Library : ${HDF5_C_STATIC_LIBRARY}")
-else()
-  find_package(HDF5 NAMES hdf5 COMPONENTS C shared NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake)
-  list(APPEND AMS_APP_LIBRARIES ${HDF5_C_SHARED_LIBRARY})
-  message(STATUS "HDF5 Shared Library : ${HDF5_C_SHARED_LIBRARY}")
-endif()
+    find_package(HDF5 NAMES hdf5 COMPONENTS C static NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake)
+    list(APPEND AMS_APP_LIBRARIES ${HDF5_C_STATIC_LIBRARY})
+    message(STATUS "HDF5 Static Library : ${HDF5_C_STATIC_LIBRARY}")
+  else()
+    find_package(HDF5 NAMES hdf5 COMPONENTS C shared NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake)
+    list(APPEND AMS_APP_LIBRARIES ${HDF5_C_SHARED_LIBRARY})
+    message(STATUS "HDF5 Shared Library : ${HDF5_C_SHARED_LIBRARY}")
+  endif()
+  if (NOT HDF5_WITH_ZLIB STREQUAL "")
+    list(APPEND AMS_APP_LIBRARIES ${HDF5_WITH_ZLIB})
+  endif()
   list(APPEND AMS_APP_INCLUDES ${HDF5_INCLUDE_DIR})
   list(APPEND AMS_APP_DEFINES "-D__ENABLE_HDF5__")
   message(STATUS "HDF5 Include directories: ${HDF5_INCLUDE_DIR}")
@@ -184,7 +191,7 @@ list(APPEND AMS_APP_LIBRARIES umpire)
 list(APPEND AMS_APP_INCLUDES ${UMPIRE_INCLUDE_DIR})
 
 find_package(nlohmann_json REQUIRED)
-list(APPEND AMS_APP_LIBRARIES nlohmann_json::nlohmann_json)
+list(APPEND AMS_APP_PRIVATE_LIBRARIES nlohmann_json::nlohmann_json)
 
 # ------------------------------------------------------------------------------
 find_package(Threads REQUIRED)
@@ -194,16 +201,50 @@ if (WITH_TORCH)
   find_package(Torch REQUIRED)
   # This is annoying, torch populates all my cuda flags
   # and resets them
-  set(CMAKE_CUDA_FLAGS "")
+  # set(CMAKE_CUDA_FLAGS "")
   set(CMAKE_CUDA_ARCHITECTURES ON)
 
-  list(APPEND AMS_APP_INCLUDES "${TORCH_INCLUDE_DIRS}")
-  list(APPEND AMS_APP_LIBRARIES "${TORCH_LIBRARIES}")
-
+  #get_target_property(torch_interface_system_includes
+  #                    torch INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
+  #if ( torch_interface_system_includes )
+  #    list(APPEND AMS_APP_INCLUDES ${torch_interface_system_includes})
+  #endif()
+  #
+  #get_target_property(torch_interface_includes
+  #                    torch INTERFACE_INCLUDE_DIRECTORIES)
+  #if ( torch_interface_includes )
+  #    list(APPEND AMS_APP_INCLUDES ${torch_interface_includes})
+  #endif()
+  #
+  #get_target_property(torch_interface_defines
+  #                    torch INTERFACE_COMPILE_DEFINITIONS)
+  #if ( troch_interface_defines )
+  #    list(APPEND AMS_APP_DEFINES ${torch_interface_defines})
+  #endif()
+  #
+  #get_target_property(torch_interface_compile_options
+  #                    torch INTERFACE_COMPILE_OPTIONS)
+  #if ( torch_interface_compile_options )
+  #    list(APPEND AMS_APP_DEFINES ${torch_interface_compile_options})
+  #endif()
+  #
+  #get_target_property(_interface_link_directories
+  #                    ${arg_FROM} INTERFACE_LINK_DIRECTORIES)
+  #if ( _interface_link_directories )
+  #    target_link_directories( ${arg_TO} ${_scope} ${_interface_link_directories})
+  #endif()
+  #
+  #get_target_property(torch_interface_link_libraries
+  #                    torch INTERFACE_LINK_LIBRARIES)
+  #if ( torch_interface_link_libraries )
+  #    list(APPEND AMS_APP_LIBRARIES "${torch_interface_link_libraries}")
+  #endif()
+
+  list(APPEND AMS_TORCH_LIBRARY torch)
   list(APPEND AMS_APP_DEFINES "-D__ENABLE_TORCH__")
-  set(BLA_VENDER OpenBLAS)
-  find_package(BLAS REQUIRED)
-  list(APPEND AMS_APP_LIBRARIES "${BLAS_LIBRARIES}")
+  #set(BLA_VENDER OpenBLAS)
+  #find_package(BLAS REQUIRED)
+  #list(APPEND AMS_APP_LIBRARIES "${BLAS_LIBRARIES}")
 endif()
 
 # ------------------------------------------------------------------------------
@@ -256,6 +297,98 @@ if (WITH_PERFFLOWASPECT)
 endif()
 
 
+macro(inherit_target_nostatic)
+    set(options)
+    set(singleValueArgs TO FROM OBJECT)
+    set(multiValueArgs)
+
+    # Parse the arguments
+    cmake_parse_arguments(arg "${options}" "${singleValueArgs}"
+                        "${multiValueArgs}" ${ARGN} )
+
+    # Check arguments
+    if ( NOT DEFINED arg_TO )
+        message( FATAL_ERROR "Must provide a TO argument to the 'blt_inherit_target' macro" )
+    endif()
+
+    if ( NOT DEFINED arg_FROM )
+        message( FATAL_ERROR "Must provide a FROM argument to the 'blt_inherit_target' macro" )
+    endif()
+
+    set(_scope INTERFACE)
+
+    get_target_property(_interface_system_includes
+                        ${arg_FROM} INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
+    if ( _interface_system_includes )
+        target_include_directories(${arg_TO} SYSTEM ${_scope} ${_interface_system_includes})
+    endif()
+
+    get_target_property(_interface_includes
+                        ${arg_FROM} INTERFACE_INCLUDE_DIRECTORIES)
+    if ( _interface_includes )
+        target_include_directories(${arg_TO} ${_scope} ${_interface_includes})
+    endif()
+
+    get_target_property(_interface_defines
+                        ${arg_FROM} INTERFACE_COMPILE_DEFINITIONS)
+    if ( _interface_defines )
+        target_compile_definitions( ${arg_TO} ${_scope} ${_interface_defines})
+    endif()
+
+    if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0" )
+        get_target_property(_interface_link_options
+                            ${arg_FROM} INTERFACE_LINK_OPTIONS)
+        if ( _interface_link_options )
+            target_link_options( ${arg_TO} ${_scope} ${_interface_link_options})
+        endif()
+    endif()
+
+    get_target_property(_interface_compile_options
+                        ${arg_FROM} INTERFACE_COMPILE_OPTIONS)
+    if ( _interface_compile_options )
+        target_compile_options( ${arg_TO} ${_scope} ${_interface_compile_options})
+    endif()
+
+    if ( NOT arg_OBJECT )
+        #get_target_property(_interface_link_directories
+        #                    ${arg_FROM} INTERFACE_LINK_DIRECTORIES)
+        #if ( _interface_link_directories )
+        #    target_link_directories( ${arg_TO} ${_scope} ${_interface_link_directories})
+        #endif()
+
+        #get_target_property(_interface_link_libraries
+        #                    ${arg_FROM} INTERFACE_LINK_LIBRARIES)
+        #if ( _interface_link_libraries )
+        #    target_link_libraries( ${arg_TO} ${_scope} ${_interface_link_libraries})
+        #endif()
+    endif()
+
+endmacro(inherit_target_nostatic)
+
+
+if (EXCLUDE_STATIC_LIBS)
+   set(NONSTATIC_AMS_APP_INTERFACE_LIBRARIES "")
+   set(NONSTATIC_AMS_APP_PRIVATE_LIBRARIES "")
+   foreach (THIS_LIB ${AMS_APP_LIBRARIES})
+     list(APPEND NONSTATIC_AMS_APP_INTERFACE_LIBRARIES ${THIS_LIB})
+     if (TARGET ${THIS_LIB})
+       get_target_property(target_type ${THIS_LIB} TYPE)
+       if (target_type STREQUAL STATIC_LIBRARY)
+          add_library("${THIS_LIB}::nostatic" INTERFACE IMPORTED)
+          inherit_target_nostatic(TO "${THIS_LIB}::nostatic" FROM ${THIS_LIB})   
+          list(APPEND NONSTATIC_AMS_APP_PRIVATE_LIBRARIES "${THIS_LIB}::nostatic")
+       else()
+          list(APPEND NONSTATIC_AMS_APP_PRIVATE_LIBRARIES ${THIS_LIB})
+       endif()
+     else()
+       get_filename_component(THIS_EXT ${THIS_LIB} EXT)
+       if (NOT ".a" STREQUAL "${THIS_EXT}")
+          list(APPEND NONSTATIC_AMS_APP_PRIVATE_LIBRARIES ${THIS_LIB})
+       endif()
+     endif()
+   endforeach()
+endif()
+
 add_subdirectory(src)
 
 # ------------------------------------------------------------------------------

examples/main.cpp

@@ -219,7 +219,7 @@ int run(const char *device_name,
     std::default_random_engine generator;
     std::normal_distribution<double> distribution(avg, stdDev);
     threshold = distribution(generator);
-#ifdef __ENABLE_MPI__
+#ifdef __AMS_ENABLE_MPI__
     if (wS > 1) {
       if (rId == 0) {
         for (int i = 1; i < wS; i++) {

src/AMSlib/AMS.cpp

@@ -595,7 +595,7 @@ AMSExecutor AMSCreateExecutor(AMSCAbstrModel model,
   }
 }
 
-#ifdef __ENABLE_MPI__
+#ifdef __AMS_ENABLE_MPI__
 AMSExecutor AMSCreateDistributedExecutor(AMSCAbstrModel model,
                                          AMSDType data_type,
                                          AMSResourceType resource_type,

src/AMSlib/CMakeLists.txt

@@ -50,7 +50,14 @@ target_include_directories(AMS PUBLIC
   $<INSTALL_INTERFACE:include/>)
 target_include_directories(AMS PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
 target_link_directories(AMS PUBLIC ${AMS_APP_LIB_DIRS})
-target_link_libraries(AMS PUBLIC ${AMS_APP_LIBRARIES} stdc++fs)
+if (EXCLUDE_STATIC_LIBS)
+   target_link_libraries(AMS PRIVATE ${NONSTATIC_AMS_APP_PRIVATE_LIBRARIES} ${AMS_APP_PRIVATE_LIBRARIES} ${AMS_TORCH_LIBRARY} stdc++fs stdc++ /usr/tce/packages/intel/intel-2022.1.0/compiler/2022.1.0/linux/compiler/lib/intel64_lin/libintlc.so m)
+   target_link_libraries(AMS INTERFACE ${NONSTATIC_AMS_APP_INTERFACE_LIBRARIES} stdc++fs stdc++ /usr/tce/packages/intel/intel-2022.1.0/compiler/2022.1.0/linux/compiler/lib/intel64_lin/libintlc.so m)
+   target_link_options(AMS PRIVATE "-Wl,--unresolved-symbols=ignore-all")
+else()
+   target_link_libraries(AMS PUBLIC ${AMS_APP_LIBRARIES} stdc++fs stdc++ /usr/tce/packages/intel/intel-2022.1.0/compiler/2022.1.0/linux/compiler/lib/intel64_lin/libintlc.so m)
+   target_link_libraries(AMS PRIVATE ${AMS_APP_PRIVATE_LIBRARIES})
+endif()
 
 #-------------------------------------------------------------------------------
 # create the configuration header file with the respective information
@@ -83,7 +90,7 @@ install(TARGETS AMS
         DESTINATION lib)
 
 install(EXPORT AMSTargets
-  FILE AMS.cmake
+  FILE AMSConfig.cmake
   DESTINATION lib/cmake/AMS)
 
 install(FILES ${PROJECT_BINARY_DIR}/include/AMS.h DESTINATION include)

src/AMSlib/include/AMS.h

@@ -20,14 +20,13 @@
 #define CALIPER(stmt)
 #endif
 
-#ifdef __ENABLE_MPI__
+#ifdef __AMS_ENABLE_MPI__
 #include <mpi.h>
 #define MPI_CALL(stmt)                                                         \
   if (stmt != MPI_SUCCESS) {                                                   \
     fprintf(stderr, "Error in MPI-Call (File: %s, %d)\n", __FILE__, __LINE__); \
   }
 #else
-typedef void *MPI_Comm;
 #define MPI_CALL(stm)
 #endif
 
@@ -78,7 +77,7 @@ AMSExecutor AMSCreateExecutor(AMSCAbstrModel model,
                               int process_id,
                               int world_size);
 
-#ifdef __ENABLE_MPI__
+#ifdef __AMS_ENABLE_MPI__
 AMSExecutor AMSCreateDistributedExecutor(AMSCAbstrModel model,
                                          AMSDType data_type,
                                          AMSResourceType resource_type,

src/AMSlib/wf/basedb.hpp

@@ -1029,7 +1029,7 @@ class AMSMessage
         _data(nullptr),
         _total_size(0)
   {
-#ifdef __ENABLE_MPI__
+#ifdef __AMS_ENABLE_MPI__
     MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &_rank));
 #endif
     AMSMsgHeader header(_rank,
@@ -1072,7 +1072,7 @@ class AMSMessage
     auto header = AMSMsgHeader::decode(data);
 
     int current_rank = 0;
-#ifdef __ENABLE_MPI__
+#ifdef __AMS_ENABLE_MPI__
     MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &current_rank));
 #endif
     _rank = header.mpi_rank;
@@ -1234,7 +1234,7 @@ class RMQConsumerHandler final : public AMQP::LibEventHandler
         _messages(std::make_shared<std::vector<inbound_msg>>()),
         _channel(nullptr)
   {
-#ifdef __ENABLE_MPI__
+#ifdef __AMS_ENABLE_MPI__
     MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &_rank));
 #endif
   }
@@ -1473,7 +1473,7 @@ class RMQConsumer
               std::string queue)
       : _rank(0), _queue(queue), _cacert(cacert), _handler(nullptr)
   {
-#ifdef __ENABLE_MPI__
+#ifdef __AMS_ENABLE_MPI__
     MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &_rank));
 #endif
 #ifdef EVTHREAD_USE_PTHREADS_IMPLEMENTED
@@ -1618,7 +1618,7 @@ class RMQPublisherHandler final : public AMQP::LibEventHandler
         _channel(nullptr),
         _rchannel(nullptr)
   {
-#ifdef __ENABLE_MPI__
+#ifdef __AMS_ENABLE_MPI__
     MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &_rank));
 #endif
     established = establish_connection.get_future();
@@ -2023,7 +2023,7 @@ class RMQPublisher
         _handler(nullptr),
         _buffer_msg(std::move(msgs_to_send))
   {
-#ifdef __ENABLE_MPI__
+#ifdef __AMS_ENABLE_MPI__
     MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &_rank));
 #endif
 #ifdef EVTHREAD_USE_PTHREADS_IMPLEMENTED