WIP: Broadcast *.so files

run_utils/lib_on_ram_disc.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+mkdir /dev/shm/"$USER"
+ldd ./icon_gpu | sed 's/\t/ /g' > temp.tmp
+cat temp.tmp | sed 's/.*=> //' | sed 's/ (.*//' | sed 's/^ *//' > temp2.tmp
+cat temp.tmp | sed 's/ =>.*//' | sed 's/.*\///' | sed "s/^[ \t]*/\/dev\/shm\/$USER\//" | sed 's/ *(.*//' > temp3.tmp
+paste temp2.tmp temp3.tmp > bcast_files.sh

run_utils/mybcast.c

@@ -0,0 +1,53 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <mpi.h>
+
+int main(int argc, char *argv[]) {
+  FILE *fhandle, *mfhandle;
+  int mysize, mpi_rank, ret = 1;
+  char line1[10000];
+  char line2[10000];
+  struct stat st = {0};
+  void *ptr = malloc(1024*1024*1024);
+  MPI_Init(&argc, &argv);
+  MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+  if (stat(argv[1], &st) == -1) {
+    mkdir(argv[1], 0700);
+  }
+  if (mpi_rank == 0) {
+    mfhandle = fopen(argv[2], "r");
+  }
+  while (ret > 0) {
+    if (mpi_rank == 0) {
+      ret = fscanf(mfhandle, "%s %s", line1, line2);
+    }
+    MPI_Bcast(&ret, 1, MPI_INT, 0, MPI_COMM_WORLD);
+    if (ret > 0) {
+      MPI_Bcast(line1, 10000, MPI_CHAR, 0, MPI_COMM_WORLD);
+      MPI_Bcast(line2, 10000, MPI_CHAR, 0, MPI_COMM_WORLD);
+      if (mpi_rank == 0) {
+        fhandle = fopen(line1, "r");
+        if (fhandle) {
+          mysize = fread(ptr, 1, 1024*1024*1024, fhandle);
+          fclose(fhandle);
+        } else {
+          mysize = 0;
+        }
+      }
+      MPI_Bcast(&mysize, 1, MPI_INT, 0, MPI_COMM_WORLD);
+      if (mysize > 0 && mysize < 1024*1024*1024) {
+        MPI_Bcast(ptr, mysize, MPI_CHAR, 0, MPI_COMM_WORLD);
+        fhandle = fopen(line2, "w");
+        if (fhandle) {
+          fwrite(ptr, 1, mysize, fhandle);
+          fclose(fhandle);
+        }
+      }
+    }
+  }
+  MPI_Finalize();
+  return 0;
+}

run_utils/run_tools.sh

@@ -49,6 +49,16 @@ set_environment(){
    ulimit -s unlimited
    ulimit -c 0
 
+   # Set up the broadcast of dynamic share libraries
+   # -----------------------------------------------
+   ../run_utils/lib_on_ram_disc.sh
+   cat ./bcast_files.sh
+   mpicc ../run_utils/mybcast.c -o mybcast
+   srun --ntasks-per-node=1 ./mybcast /dev/shm/$USER ./bcast_files.sh
+   ls -l /dev/shm/$USER
+   export LD_LIBRARY_PATH="/dev/shm/$USER:$LD_LIBRARY_PATH"
+   ldd ./icon_gpu
+
    # Dump SLURM environment variables
    # --------------------------------
    set | grep SLURM