From 02d74c88512fcd343a7d1ce9b69fc8eb6c8533c0 Mon Sep 17 00:00:00 2001 From: Andreas Jocksch Date: Wed, 14 Jan 2026 16:00:38 +0100 Subject: [PATCH 1/3] broadcast of shared object files --- run_utils/lib_on_ram_disc.sh | 7 +++++ run_utils/mybcast.c | 53 ++++++++++++++++++++++++++++++++++++ run_utils/run_tools.sh | 10 +++++++ 3 files changed, 70 insertions(+) create mode 100755 run_utils/lib_on_ram_disc.sh create mode 100644 run_utils/mybcast.c diff --git a/run_utils/lib_on_ram_disc.sh b/run_utils/lib_on_ram_disc.sh new file mode 100755 index 0000000..5c8f9c6 --- /dev/null +++ b/run_utils/lib_on_ram_disc.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +mkdir /dev/shm/"$USER" +ldd ./icon_gpu | sed 's/\t/ /g' > temp.tmp +cat temp.tmp | sed 's/.*=> //' | sed 's/ (.*//' | sed 's/^ *//' > temp2.tmp +cat temp.tmp | sed 's/ =>.*//' | sed 's/.*\///' | sed "s/^[ \t]*/\/dev\/shm\/$USER\//" | sed 's/ *(.*//' > temp3.tmp +paste temp2.tmp temp3.tmp > bcast_files.sh diff --git a/run_utils/mybcast.c b/run_utils/mybcast.c new file mode 100644 index 0000000..de6613b --- /dev/null +++ b/run_utils/mybcast.c @@ -0,0 +1,53 @@ +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) { + FILE *fhandle, *mfhandle; + int mysize, mpi_rank, ret = 1; + char line1[10000]; + char line2[10000]; + struct stat st = {0}; + void *ptr = malloc(1024*1024*1024); + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + if (stat(argv[1], &st) == -1) { + mkdir(argv[1], 0700); + } + if (mpi_rank == 0) { + mfhandle = fopen(argv[2], "r"); + } + while (ret > 0) { + if (mpi_rank == 0) { + ret = fscanf(mfhandle, "%s %s", line1, line2); + } + MPI_Bcast(&ret, 1, MPI_INT, 0, MPI_COMM_WORLD); + if (ret > 0) { + MPI_Bcast(line1, 10000, MPI_CHAR, 0, MPI_COMM_WORLD); + MPI_Bcast(line2, 10000, MPI_CHAR, 0, MPI_COMM_WORLD); + if (mpi_rank == 0) { + fhandle = fopen(line1, "r"); + if (fhandle) { + mysize = fread(ptr, 1, 1024*1024*1024, fhandle); + fclose(fhandle); + } else { + mysize = 0; + } + } + MPI_Bcast(&mysize, 1, MPI_INT, 0, MPI_COMM_WORLD); + if (mysize > 0 && mysize < 1024*1024*1024) { + MPI_Bcast(ptr, mysize, MPI_CHAR, 0, MPI_COMM_WORLD); + fhandle = fopen(line2, "w"); + if (fhandle) { + fwrite(ptr, 1, mysize, fhandle); + fclose(fhandle); + } + } + } + } + MPI_Finalize(); + return 0; +} diff --git a/run_utils/run_tools.sh b/run_utils/run_tools.sh index 99a1627..74aa2c9 100644 --- a/run_utils/run_tools.sh +++ b/run_utils/run_tools.sh @@ -49,6 +49,16 @@ set_environment(){ ulimit -s unlimited ulimit -c 0 + # Set up the broadcast of dynamic share libraries + # ----------------------------------------------- + ../run_utils/lib_on_ram_disc.sh + cat ./bcast_files.sh + mpicc ../run_utils/mybcast.c -o mybcast + srun --ntasks-per-node=1 ./mybcast /dev/shm/$USER ./bcast_files.sh + ls -l /dev/shm/$USER + export LD_LIBRARY_PATH="/dev/shm/$USER:$LD_LIBRARY_PATH" + ldd ./icon_gpu + # Dump SLURM environment variables # -------------------------------- set | grep SLURM From 6050aee3aacd9dad425acbd99412ad456382d304 Mon Sep 17 00:00:00 2001 From: Nathan Date: Wed, 14 Jan 2026 16:06:38 +0100 Subject: [PATCH 2/3] update to main --- experiments/namelists.sh | 3 ++- full_build.o | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 full_build.o diff --git a/experiments/namelists.sh b/experiments/namelists.sh index b9147e7..83d01a8 100644 --- a/experiments/namelists.sh +++ b/experiments/namelists.sh @@ -128,7 +128,8 @@ coupling: #weight_file_name: w_oce2atm.nc - <<: [ *oce2atm, *conserv_interp_stack ] coupling_period: ${atm_oce_coupling_timestep} - field: [surface_velocity_bundle] + field: [eastward_sea_water_velocity, + northward_sea_water_velocity] #weight_file_name: w_riv2oce.nc - <<: [ *atm2oce, *spmap_interp_stack ] coupling_period: ${atm_oce_coupling_timestep} diff --git a/full_build.o b/full_build.o new file mode 100644 index 0000000..0605ff4 --- /dev/null +++ b/full_build.o @@ -0,0 +1,9 @@ +[CAO build] Using build scripts from /capstor/scratch/cscs/nbeech/coupled_setup/EXCLAIM_coupled_setup/build_utils +[CAO build] Building in /dev/shm/nbeech/coupled_setup +[CAO build] Getting ICON +Cloning into 'icon-hybrid'... +git@gitlab.dkrz.de: Permission denied (publickey). +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. From 6e152a2f71b97498b8961ca241ea3914865e6aa8 Mon Sep 17 00:00:00 2001 From: Andreas Jocksch Date: Tue, 20 Jan 2026 15:50:46 +0100 Subject: [PATCH 3/3] deleted full_build --- full_build.o | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 full_build.o diff --git a/full_build.o b/full_build.o deleted file mode 100644 index 0605ff4..0000000 --- a/full_build.o +++ /dev/null @@ -1,9 +0,0 @@ -[CAO build] Using build scripts from /capstor/scratch/cscs/nbeech/coupled_setup/EXCLAIM_coupled_setup/build_utils -[CAO build] Building in /dev/shm/nbeech/coupled_setup -[CAO build] Getting ICON -Cloning into 'icon-hybrid'... -git@gitlab.dkrz.de: Permission denied (publickey). -fatal: Could not read from remote repository. - -Please make sure you have the correct access rights -and the repository exists.