Merge changes from gadi and setonix together

Author: Julian Giordani

magnus_baremetal_go.sh renamed to deprecated/magnus_baremetal_go.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -l
+ 
+# load Singularity
+module load singularity
+module load openmpi/4.1.4
+ 
+
+export singularityDir=/g/data/m18/software/underworld/
+
+# Define the container to use
+export containerImage=$singularityDir/underworld-215.sif
+ 
+env
+cat timed_model.py
+echo ""
+echo "---------- Running Job ----------"
+echo ""
+export TIME_LAUNCH_MPI=`date +%s%N | cut -b1-13`
+mpiexec -n $PBS_NCPUS singularity exec $containerImage bash -c "TIME_LAUNCH_PYTHON=\`date +%s%N | cut -b1-13\` python3 timed_model.py"

magnus_container_go.sh renamed to deprecated/magnus_container_go.sh

@@ -1,19 +1,21 @@
 #!/bin/bash
-export JOBS="4 5 6 7 8"
-export UW_NAME="Strong_Scaling_Init_Test"
-export SCALING_TYPE=2  # 1=weak, 2=strong
+export JOBS="1 2 4 6 8"
+#export JOBS="1 2 4 6 8 10 12 14 16 18 20 22"
+#export JOBS="22 22"
+export UW_NAME="2.15.0_WeakScaling"
+export SCALING_TYPE=1  # 1=weak, 2=strong
 
 export UW_MODEL="SolDB3d"  #  "SolDB3d" for dim3, though penalty method probably needed for q1.
 export PICKLENAME=None   #"SolDB3d_Gadi_1e-11.pickle" #"conv_test_results_high_res_tighter_take2.pickle"  # set to "None" to disable conv testing
 export UW_ENABLE_IO="0"
 
-export WALLTIME="00:15:00"
-export ACCOUNT="director2186"
+export WALLTIME="00:10:00"
+export ACCOUNT="m18"
 export QUEUE="normal" # normal or express  
 
 export UW_ORDER=1
 export UW_DIM=3
-export SCALING_BASE=256
+export SCALING_BASE=16
 
 # Test style - UW_MAX_ITS (+ve, recommended >100): Fixed work, (-ve): Accuracy (UW_SOL_TERANCE is used)
 export UW_MAX_ITS=50 # set to negative for accuracy test, positive for fixed iterative work irrepective of result fidelity

gadi_container_go.sh

@@ -42,14 +42,13 @@ do
    export NTASKS="$((${i}*${i}*${i}))"
    export EXPORTVARS="UW_RESOLUTION,NTASKS,UW_ENABLE_IO,UW_ORDER,UW_DIM,UW_SOL_TOLERANCE,UW_MAX_ITS,UW_PENALTY,UW_MODEL,PICKLENAME"
    if [ $BATCH_SYS == "PBS" ] ; then
-      # memory requirement guess: 3GB * nprocs
-      MEMORY="$((4*${NTASKS}))GB"
       PBSTASKS=`python3 <<<"print((int(${NTASKS}/48) + (${NTASKS} % 48 > 0))*48)"`  # round up to nearest 48 as required by nci
-      CMD="qsub -v ${EXPORTVARS} -N ${NAME} -l storage=gdata/m18,ncpus=${PBSTASKS},mem=${MEMORY},walltime=${WALLTIME},wd -P ${ACCOUNT} -q ${QUEUE} gadi_baremetal_go.sh"
+      # memory requirement guess: 3GB * nprocs
+      MEMORY="$((3*${PBSTASKS}))GB"
+      CMD="qsub -v ${EXPORTVARS} -N ${NAME} -l storage=gdata/m18+gdata/q97,ncpus=${PBSTASKS},mem=${MEMORY},walltime=${WALLTIME},wd -P ${ACCOUNT} -q ${QUEUE} gadi_container_go.sh"
       echo ${CMD}
       ${CMD}
    else
-      #export IMAGE=/group/m18/singularity/underworld/underworld2_2.11.0b.sif
       #export IMAGE=/group/m18/singularity/underworld/underworld2_2.10.0b_rc.sif
       #export IMAGE=/group/m18/singularity/underworld/underworld2_v29.sif
       if [[ "$QUEUE" == "express" ]] ; then
@@ -59,7 +58,11 @@ do
       fi
       export OUTNAME="Res_"${UW_RESOLUTION}"_Nproc_"${NTASKS}"_JobID_"%j".out"
 
-      CMD="sbatch --export=${EXPORTVARS} --job-name=${NAME} --output=${OUTNAME} --ntasks=${NTASKS} --time=${WALLTIME} --account=${ACCOUNT} setonix_baremetal_go.sh"
+      # Container
+      #CMD="sbatch --export=IMAGE,${EXPORTVARS} --job-name=${NAME} --output=${OUTNAME} --ntasks=${NTASKS} --time=${WALLTIME} --account=${ACCOUNT} --partition=${QUEUE} setonix_container_go.sh"
+
+      # Baremetal
+      CMD="sbatch --export=IMAGE,${EXPORTVARS} --job-name=${NAME} --output=${OUTNAME} --ntasks=${NTASKS} --time=${WALLTIME} --account=${ACCOUNT}  setonix_baremetal_go.sh"
       echo ${CMD}
       ${CMD}
    fi

params.sh

@@ -0,0 +1,22 @@
+#!/bin/bash -l
+ 
+# load Singularity
+module load singularity/3.8.6
+ 
+export UW_VIS_PORT=0
+ 
+# as per
+# https://support.pawsey.org.au/documentation/pages/viewpage.action?pageId=116131367#UsewithSingularity-RunningPythonandR
+# we unset all the host python-related ENV vars
+unset $( env | grep ^PYTHON | cut -d = -f 1 | xargs )
+
+scontrol show job ${SLURM_JOBID} -ddd
+env
+cat timed_model.py
+echo ""
+echo "---------- Running Job ----------"
+echo ""
+export TIME_LAUNCH_MPI=`date +%s%N | cut -b1-13`
+srun --export=all -u -n ${SLURM_NTASKS} singularity exec -B ${PWD}:/work $IMAGE bash -c "cd work ; TIME_LAUNCH_PYTHON=\`date +%s%N | cut -b1-13\` python3 timed_model.py"
+# execute
+#srun --export=all -u -n $SLURM_NTASKS singularity exec -B ${PWD}:/work $containerImage bash -c "whoami; cd /work/; python3 $model"