src: Adds multi-init and multi-finalize support

mpp/shmem_c_func.h4

@@ -26,6 +26,7 @@ dnl
 SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmem_init(void);
 SHMEM_FUNCTION_ATTRIBUTES int SHPRE()shmem_init_thread(int requested, int *provided);
 SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmem_query_thread(int *provided);
+SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmem_query_initialized(int *initialized);
 SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmem_finalize(void);
 SHMEM_FUNCTION_ATTRIBUTES int SHPRE()shmem_n_pes(void);
 SHMEM_FUNCTION_ATTRIBUTES int SHPRE()shmem_my_pe(void);

src/init.c

@@ -89,7 +89,7 @@ int shmem_external_heap_device = -1;
 
 int shmem_internal_my_pe = -1;
 int shmem_internal_num_pes = -1;
-int shmem_internal_initialized = 0;
+int shmem_internal_init_counter = 0;
 int shmem_internal_finalized = 0;
 int shmem_internal_initialized_with_start_pes = 0;
 int shmem_internal_global_exit_called = 0;
@@ -138,13 +138,19 @@ shmem_internal_randr_fini(void)
 static void
 shmem_internal_shutdown(void)
 {
-    if (!shmem_internal_initialized ||
+    if (!shmem_internal_init_counter ||
         shmem_internal_finalized) {
         return;
     }
 
     shmem_internal_barrier_all();
 
+    shmem_internal_init_counter -= 1;
+
+    if (shmem_internal_init_counter > 0) {
+        return;
+    }
+
     shmem_internal_finalized = 1;
 
     shmem_internal_team_fini();
@@ -165,12 +171,12 @@ shmem_internal_shutdown(void)
 static void
 shmem_internal_shutdown_atexit(void)
 {
-    if ( shmem_internal_initialized && !shmem_internal_finalized &&
+    if ( shmem_internal_init_counter && !shmem_internal_finalized &&
          !shmem_internal_initialized_with_start_pes && !shmem_internal_global_exit_called &&
          shmem_internal_my_pe == 0) {
         RAISE_WARN_STR("shutting down without a call to shmem_finalize");
     }
-
+    shmem_internal_init_counter = 1;
     shmem_internal_shutdown();
 }
 
@@ -518,7 +524,6 @@ shmem_internal_heap_postinit(void)
     randr_initialized = 1;
 
     atexit(shmem_internal_shutdown_atexit);
-    shmem_internal_initialized = 1;
 
     /* finish up */
 #ifndef USE_PMIX
@@ -554,12 +559,15 @@ shmem_internal_init(int tl_requested, int *tl_provided)
 {
     int ret;
 
-    ret = shmem_internal_heap_preinit(tl_requested, tl_provided);
-    if (ret) goto cleanup;
+    if (shmem_internal_init_counter == 0) {
+        ret = shmem_internal_heap_preinit(tl_requested, tl_provided);
+        if (ret) goto cleanup;
 
-    ret = shmem_internal_heap_postinit();
-    if (ret) goto cleanup;
+        ret = shmem_internal_heap_postinit();
+        if (ret) goto cleanup;
+    }
 
+    shmem_internal_init_counter += 1;
     return 0;
 
  cleanup:

src/init_c.c

@@ -62,7 +62,7 @@
 void SHMEM_FUNCTION_ATTRIBUTES
 start_pes(int npes)
 {
-    if (!shmem_internal_initialized) {
+    if (!shmem_internal_init_counter) {
         shmem_internal_start_pes(npes);
     }
 }
@@ -73,10 +73,6 @@ shmem_init(void)
 {
     int tl_provided, ret;
 
-    if (shmem_internal_initialized) {
-        RAISE_ERROR_STR("attempt to reinitialize library");
-    }
-
     ret = shmem_internal_init(SHMEM_THREAD_SINGLE, &tl_provided);
     if (ret) abort();
 }
@@ -87,7 +83,7 @@ shmemx_heap_preinit(void)
 {
     int tl_provided, ret;
 
-    if (shmem_internal_initialized) {
+    if (shmem_internal_init_counter) {
         RAISE_ERROR_STR("attempt to reinitialize library");
     }
 
@@ -108,9 +104,6 @@ int SHMEM_FUNCTION_ATTRIBUTES
 shmem_init_thread(int tl_requested, int *tl_provided)
 {
     int ret;
-    if (shmem_internal_initialized) {
-        RAISE_ERROR_STR("attempt to reinitialize library");
-    }
 
     ret = shmem_internal_init(tl_requested, tl_provided);
     return ret;
@@ -121,7 +114,7 @@ int SHMEM_FUNCTION_ATTRIBUTES
 shmemx_heap_preinit_thread(int tl_requested, int *tl_provided)
 {
     int ret;
-    if (shmem_internal_initialized) {
+    if (shmem_internal_init_counter) {
         RAISE_ERROR_STR("attempt to reinitialize library");
     }
 
@@ -138,6 +131,11 @@ shmem_query_thread(int *provided)
     *provided = shmem_internal_thread_level;
 }
 
+void SHMEM_FUNCTION_ATTRIBUTES
+shmem_query_initialized(int *initialized)
+{
+    *initialized = shmem_internal_init_counter;
+}
 
 void SHMEM_FUNCTION_ATTRIBUTES
 shmem_global_exit(int status)

src/init_f.c

@@ -26,7 +26,7 @@ void SHMEM_FUNCTION_ATTRIBUTES FC_START_PES(fortran_integer_t *npes);
 void
 FC_START_PES(fortran_integer_t *npes)
 {
-    if (shmem_internal_initialized) {
+    if (shmem_internal_init_counter) {
         RAISE_ERROR_STR("attempt to reinitialize library");
     }
 
@@ -41,7 +41,7 @@ FC_SHMEM_INIT(void)
 {
     int tl_provided;
 
-    if (shmem_internal_initialized) {
+    if (shmem_internal_init_counter) {
         RAISE_ERROR_STR("attempt to reinitialize library");
     }
 

src/shmem_internal.h

@@ -33,7 +33,7 @@
 extern int shmem_internal_my_pe;
 extern int shmem_internal_num_pes;
 
-extern int shmem_internal_initialized;
+extern int shmem_internal_init_counter;
 extern int shmem_internal_finalized;
 extern int shmem_internal_thread_level;
 
@@ -189,7 +189,7 @@ extern hwloc_topology_t shmem_internal_topology;
 #ifdef ENABLE_ERROR_CHECKING
 #define SHMEM_ERR_CHECK_INITIALIZED()                                    \
     do {                                                                 \
-        if (!shmem_internal_initialized) {                               \
+        if (!shmem_internal_init_counter) {                               \
             RETURN_ERROR_STR(PACKAGE_NAME " library not initialized\n"); \
             abort();                                                     \
         }                                                                \

src/symmetric_heap_c.c

@@ -439,7 +439,7 @@ shmem_malloc_with_hints(size_t size, long hints)
 void SHMEM_FUNCTION_ATTRIBUTES
 shmemx_heap_create(void *base, size_t size, int device_type, int device_index) {
 
-    if (shmem_internal_initialized) {
+    if (shmem_internal_init_counter) {
         RAISE_WARN_MSG("Ignoring pre-setup. Heap already initialized\n");
         return;
     }