Gadi singularity modifications

* Fixing the weak scaling to use the right number for ncpu decomposition
* Adding gadi_container_go.sh

Author: julesghub

gadi_baremetal_go.sh

@@ -1,7 +1,11 @@
 #!/bin/bash
 
-# the follow load the full software stack and running environment on gadi
-source /g/data/m18/codes/UWGeodynamics_2.10.sh
+module load petsc/3.21.3 python3/3.11.7 openmpi/4.1.7 python3-as-python hdf5/1.12.2p
+ 
+UWENV=/g/data/m18/software/venv/uw216 # Project directory
+export PATH=${UWENV}/bin:$PATH
+export PYTHONPATH=${UWENV}/lib/python3.11/site-packages/:$PYTHONPATH
+
 env
 cat timed_model.py
 

gadi_container_go.sh

@@ -5,15 +5,21 @@ module load singularity
 module load openmpi/4.1.4
 
 
-export singularityDir=/g/data/m18/software/underworld/
+#export singularityDir=/home/565/jug565/scratch_m18/
+export singularityDir=/scratch/m18/jug565/codes/
 
 # Define the container to use
-export containerImage=$singularityDir/underworld-215.sif
+#export containerImage=$singularityDir/underworld-215.sif
+export containerImage=$singularityDir/uw-rhel.sif
 
 env
 cat timed_model.py
 echo ""
 echo "---------- Running Job ----------"
 echo ""
 export TIME_LAUNCH_MPI=`date +%s%N | cut -b1-13`
-mpiexec -n $PBS_NCPUS singularity exec $containerImage bash -c "TIME_LAUNCH_PYTHON=\`date +%s%N | cut -b1-13\` python3 timed_model.py"
+#$PBS_NCPUS vs $NTASKS
+mpiexec -n $NTASKS singularity exec -B /opt/pbs/default/lib/,/half-root/ --bind /lib64:/glib64 \
+  $containerImage \
+  bash -c "TIME_LAUNCH_PYTHON=\`date +%s%N | cut -b1-13\` LD_LIBRARY_PATH=/apps/openmpi/4.1.4/lib:/glib64:$LD_LIBRARY_PATH python3 timed_model.py"
+#mpiexec -n $PBS_NCPUS singularity exec $containerImage bash -c "TIME_LAUNCH_PYTHON=\`date +%s%N | cut -b1-13\` LD_LIBRARY_PATH=/apps/openmpi/4.1.4/lib:$LD_LIBRARY_PATH python3 timed_model.py"

params.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
-export JOBS="1 2 4 6 8"
-#export JOBS="1 2 4 6 8 10 12 14 16 18 20 22"
+#export JOBS="1 2 3"
+#export JOBS="8 10 12"
+export JOBS="1 2 3 4 5 6 8 10 12 14 18"
 #export JOBS="22 22"
-export UW_NAME="2.15.0_WeakScaling"
+export UW_NAME="UW216-container"
 export SCALING_TYPE=1  # 1=weak, 2=strong
 
 export UW_MODEL="SolDB3d"  #  "SolDB3d" for dim3, though penalty method probably needed for q1.
 export PICKLENAME=None   #"SolDB3d_Gadi_1e-11.pickle" #"conv_test_results_high_res_tighter_take2.pickle"  # set to "None" to disable conv testing
 export UW_ENABLE_IO="0"
 
-export WALLTIME="00:10:00"
+export WALLTIME="00:15:00"
 export ACCOUNT="m18"
 export QUEUE="normal" # normal or express  
 

scaling_test_job_launcher.sh

@@ -46,6 +46,7 @@ do
       # memory requirement guess: 3GB * nprocs
       MEMORY="$((3*${PBSTASKS}))GB"
       CMD="qsub -v ${EXPORTVARS} -N ${NAME} -l storage=gdata/m18+gdata/q97,ncpus=${PBSTASKS},mem=${MEMORY},walltime=${WALLTIME},wd -P ${ACCOUNT} -q ${QUEUE} gadi_container_go.sh"
+      #CMD="qsub -v ${EXPORTVARS} -N ${NAME} -l storage=gdata/m18+gdata/q97,ncpus=${PBSTASKS},mem=${MEMORY},walltime=${WALLTIME},wd -P ${ACCOUNT} -q ${QUEUE} gadi_baremetal_go.sh"
       echo ${CMD}
       ${CMD}
    else