diff --git a/run_utils/lib_on_ram_disc.sh b/run_utils/lib_on_ram_disc.sh new file mode 100755 index 0000000..5c8f9c6 --- /dev/null +++ b/run_utils/lib_on_ram_disc.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +mkdir /dev/shm/"$USER" +ldd ./icon_gpu | sed 's/\t/ /g' > temp.tmp +cat temp.tmp | sed 's/.*=> //' | sed 's/ (.*//' | sed 's/^ *//' > temp2.tmp +cat temp.tmp | sed 's/ =>.*//' | sed 's/.*\///' | sed "s/^[ \t]*/\/dev\/shm\/$USER\//" | sed 's/ *(.*//' > temp3.tmp +paste temp2.tmp temp3.tmp > bcast_files.sh diff --git a/run_utils/mybcast.c b/run_utils/mybcast.c new file mode 100644 index 0000000..de6613b --- /dev/null +++ b/run_utils/mybcast.c @@ -0,0 +1,53 @@ +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) { + FILE *fhandle, *mfhandle; + int mysize, mpi_rank, ret = 1; + char line1[10000]; + char line2[10000]; + struct stat st = {0}; + void *ptr = malloc(1024*1024*1024); + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + if (stat(argv[1], &st) == -1) { + mkdir(argv[1], 0700); + } + if (mpi_rank == 0) { + mfhandle = fopen(argv[2], "r"); + } + while (ret > 0) { + if (mpi_rank == 0) { + ret = fscanf(mfhandle, "%s %s", line1, line2); + } + MPI_Bcast(&ret, 1, MPI_INT, 0, MPI_COMM_WORLD); + if (ret > 0) { + MPI_Bcast(line1, 10000, MPI_CHAR, 0, MPI_COMM_WORLD); + MPI_Bcast(line2, 10000, MPI_CHAR, 0, MPI_COMM_WORLD); + if (mpi_rank == 0) { + fhandle = fopen(line1, "r"); + if (fhandle) { + mysize = fread(ptr, 1, 1024*1024*1024, fhandle); + fclose(fhandle); + } else { + mysize = 0; + } + } + MPI_Bcast(&mysize, 1, MPI_INT, 0, MPI_COMM_WORLD); + if (mysize > 0 && mysize < 1024*1024*1024) { + MPI_Bcast(ptr, mysize, MPI_CHAR, 0, MPI_COMM_WORLD); + fhandle = fopen(line2, "w"); + if (fhandle) { + fwrite(ptr, 1, mysize, fhandle); + fclose(fhandle); + } + } + } + } + MPI_Finalize(); + return 0; +} diff --git a/run_utils/run_tools.sh b/run_utils/run_tools.sh index d294aa4..f15a7e7 100644 --- a/run_utils/run_tools.sh +++ b/run_utils/run_tools.sh @@ -49,6 +49,16 @@ set_environment(){ ulimit -s unlimited ulimit -c 0 + # Set up the broadcast of dynamic share libraries + # ----------------------------------------------- + ../run_utils/lib_on_ram_disc.sh + cat ./bcast_files.sh + mpicc ../run_utils/mybcast.c -o mybcast + srun --ntasks-per-node=1 ./mybcast /dev/shm/$USER ./bcast_files.sh + ls -l /dev/shm/$USER + export LD_LIBRARY_PATH="/dev/shm/$USER:$LD_LIBRARY_PATH" + ldd ./icon_gpu + # Dump SLURM environment variables # -------------------------------- set | grep SLURM