scripts that run all affinity tests in one go, and the updated performance results

timing of kernel measured with MPI_Wtime() instead of measuring the entire execution externally

Author: cwsmith

README.md

@@ -20,19 +20,19 @@ Here workers are representing MPI tasks. Data is exhchanged between MPI tasks. T
 - envMvapich.sh - environment file for Mvapich2-mic
 - envMvapich2.sh - environment file for Mvapich2-mic v2.0
 - build.sh - build script 
-- getTimeImpi.sh - run script for Intel MPI
+- getTimeImpi.sh - run script for Intel MPI 4 and 5
+- getTimeImpi51.sh - run script for Intel MPI 5.1
 - getTimeMvapich.sh - run script Mvapich2-mic
 - getTimeMvapich2.sh - run script Mvapich2-mic
-- runImpi51.sh - run script for Intel MPI 5.1 (beta)
 - kernelComm.c - communication kernel
 - kernel.h - kernel header
 - mpiWork.c - mpi driver
 - README.md - this file 
 - threadWork.c - threaded mpi driver
 
 ## build
-    source env<Impi|Impi5|Mvapich|Mvapich2>.sh
-    ./build.sh <threadsPerCore=1|2|4>
+    source env<Impi|Impi51|Mvapich|Mvapich2>.sh
+    for i in 1 2 4; do ./build.sh $i; done
 
 ## run
 From an interactive idev session:
@@ -67,8 +67,4 @@ run
 ## Performance Results
 
 The following tests used MPSS 3.3 as installed on Stampede.
-
-![alt text](
-https://github.com/cwsmith/commKernel/raw/master/mpiThreadMultiplePerformanceOnPhi.png
-"Performance Results")
-
+see performance.csv

build.sh

@@ -4,7 +4,7 @@ tpc=$1
 flags="-DTHREADS_PER_CORE=$tpc -mmic -Wall -g -O3 -fno-omit-frame-pointer "
 src="kernelComm.c"
 set -x
-mpicc $flags threadWork.c $src -o threadWorkComm -lpthread $LDFLAGS $THREAD_MULTIPLE
-mpicc $flags mpiWork.c $src -o mpiWork -lpthread $LDFLAGS
+mpicc $flags threadWork.c $src -o threadWorkComm.${tpc} -lpthread $LDFLAGS $THREAD_MULTIPLE
+mpicc $flags mpiWork.c $src -o mpiWork.${tpc} -lpthread $LDFLAGS
 mpicc -DTHREAD_MULTIPLE $flags mpiWork.c $src \
-  -o mpiWorkThreadMult -lpthread $LDFLAGS $THREAD_MULTIPLE
+  -o mpiWorkThreadMult.${tpc} -lpthread $LDFLAGS $THREAD_MULTIPLE

getTimeImpi.sh

@@ -1,38 +1,42 @@
 #!/bin/bash -e
 run() {
-  /usr/bin/time -f "realTime %e" ibrun.symm -m "$1" &>> work${2}.log
+  ibrun.symm -m "$1" &>> work${2}.log
 }
 threadWorkComm() { 
   export MIC_MY_NSLOTS=1
   export MIC_PPN=1
-  run "./threadWorkComm $1" $1
+  run "./threadWorkComm.${1} $2" $2
 }
 mpiWork() { 
   export MIC_MY_NSLOTS=$1
   export MIC_PPN=$1
-  run "./mpiWork" $1
+  run "./mpiWork.${1}" $2
 }
 mpiWorkThreadMult() { 
   export MIC_MY_NSLOTS=$1
   export MIC_PPN=$1
-  run "./mpiWorkThreadMult" $1
+  run "./mpiWorkThreadMult.${1}" $2
 }
 getAvg() {
-  awk '/realTime/ {sum+=$2; cnt+=1;} END {print "average " sum/cnt}' work${1}.log 
+  t=$(awk '/realTime/ {sum+=$2; cnt+=1;} END {print sum/cnt}' work${1}.log)
+  echo -n ", $t "
 }
 
 declare -a fns
 fns[1]="threadWorkComm"
 fns[2]="mpiWork"
 fns[3]="mpiWorkThreadMult"
 
-for workers in 2 4 8 16 32; do
-  for i in "${!fns[@]}"; do   
-    cat /dev/null > work${workers}.log
-    echo -n "${fns[i]} workers $workers "
-    for j in {0..4}; do 
-      ${fns[$i]} $workers
+for fnIdx in "${!fns[@]}"; do
+  for affinity in 1 2 4; do 
+    echo -n "${fns[$fnIdx]}, $affinity"
+    for workers in 2 4 8 16 32; do
+      cat /dev/null > work${workers}.log
+      for j in {0..4}; do 
+        ${fns[$fnIdx]} $affinity $workers
+      done
+      getAvg $workers
     done
-    getAvg $workers
+    echo ""
   done
 done

getTimeImpi51.sh

@@ -0,0 +1,95 @@
+#!/bin/bash -e
+
+setup() {
+  ulimit -l unlimited
+  echo "memlock ulimit:" `ulimit -l`
+
+  for i in `env | grep I_MPI| cut -f 1 -d '='`; do unset $i; done
+  for i in `env | grep DAPL | cut -f 1 -d '='`; do unset $i; done
+  for i in `env | grep TACC | cut -f 1 -d '='`; do unset $i; done
+
+  beta=/work/01187/bmatth/intel_tools/2016_beta1
+  compiler=$beta/compilers_and_libraries_2016.0.042
+  mpi=$beta/impi/5.1.0.042
+  export SINK_LD_LIBRARY_PATH=$compiler/linux/compiler/lib/mic/:$mpi/mic/lib/:$SINK_LD_LIBRARY_PATH
+  export SINK_PATH=$mpi/mic/bin:$SINK_PATH
+
+  . $beta/bin/compilervars.sh intel64
+  . $mpi/bin64/mpivars.sh
+  export I_MPI_CC=$compiler/linux/bin/intel64_mic/icc
+  export I_MPI_CXX=$compiler/linux/bin/intel64_mic/icpc
+  export I_MPI_FC=$compiler/linux/bin/intel64_mic/ifort
+}
+
+mpirun() {
+#EDIT THESE
+  hostnp=0 #tasks per host
+  micnp=1 #tasks per accelerator
+  n_mics=1 #accelerators per host
+  app=$1
+
+#The rest of this script should be safe to ignore
+#(Give or take performance)
+  tmpdir=`mktemp -d`
+
+  hosts=()
+  for i in $SLURM_NODELIST; do
+    for j in `scontrol show hostname $i`; do
+      hosts+=($j)
+    done
+  done
+  echo "-genv I_MPI_DEBUG 5" > $tmpdir/mpi_conf
+  echo "-genv I_MPI_FABRICS 'shm:dapl'" >> $tmpdir/mpi_conf
+  echo "-genv I_MPI_FALLBACK 0" >> $tmpdir/mpi_conf
+  echo "-genv I_MPI_MIC 1" >> $tmpdir/mpi_conf
+  echo "-genv I_MPI_MIC_PREFIX ./" >> $tmpdir/mpi_conf
+  echo "-genv I_MPI_EXTRA_FILE_SYSTEM 1" >> $tmpdir/mpi_conf
+  echo "-genv I_MPI_EXTRA_FILE_SYSTEM_LLIST lustre" >> $tmpdir/mpi_conf
+  echo "-genv I_MPI_MIC_PROXY_PATH $mpi/mic/bin" >> $tmpdir/mpi_conf
+  MIC_ENVS="-genv PATH $SINK_PATH -genv LD_LIBRARY_PATH $SINK_LD_LIBRARY_PATH"
+
+  for h in "${hosts[@]}"; do
+    for i in `seq 0 $(( $n_mics-1 ))`; do
+      printf -- "$MIC_ENVS -n %d -host %s %s\n" $micnp "$h-mic$i" "$app" >> $tmpdir/mpi_conf
+    done
+    printf -- "$HOST_ENVS -n %d -host %s %s\n" $hostnp "$h-br0" "$app" >> $tmpdir/mpi_conf
+  done
+
+  cat $tmpdir/mpi_conf
+  which mpirun
+  export I_MPI_MIC=1
+  export I_MPI_MIC_PROXY_PATH=$mpi/mic/bin
+  mpiexec.hydra -iface br0 -configfile $tmpdir/mpi_conf
+  cp $tmpdir/mpi_conf ./
+  rm -rf $tmpdir
+}
+
+run() {
+  mpirun "$1" &>> work${2}.log
+}
+threadWorkComm() { 
+  run "./threadWorkComm.${1} $2" $2
+}
+getAvg() {
+  t=$(awk '/realTime/ {sum+=$2; cnt+=1;} END {print sum/cnt}' work${1}.log)
+  echo -n ", $t "
+}
+
+declare -a fns
+fns[1]="threadWorkComm"
+
+setup
+
+for fnIdx in "${!fns[@]}"; do
+  for affinity in 1 2 4; do 
+    echo -n "${fns[$fnIdx]}, $affinity"
+    for workers in 2 4 8 16 32; do
+      cat /dev/null > work${workers}.log
+      for j in {0..4}; do 
+        ${fns[$fnIdx]} $affinity $workers
+      done
+      getAvg $workers
+    done
+    echo ""
+  done
+done

getTimeMvapich.sh

@@ -1,40 +1,44 @@
 #!/bin/bash -e
 run() {
-  /usr/bin/time -f "realTime %e" ibrun.symm -m "$1" &>> work${2}.log
+  ibrun.symm -m "$1" &>> work${2}.log
 }
 threadWorkComm() { 
   export MIC_MY_NSLOTS=1
   export MIC_PPN=1
   export MV2_ENABLE_AFFINITY=0
-  run "./threadWorkComm $1" $1
+  run "./threadWorkComm.${1} $2" $2
 }
 mpiWork() { 
   export MIC_MY_NSLOTS=$1
   export MIC_PPN=$1
-  run "./mpiWork" $1
+  run "./mpiWork.${1}" $2
 }
 mpiWorkThreadMult() { 
   export MIC_MY_NSLOTS=$1
   export MIC_PPN=$1
   export MV2_ENABLE_AFFINITY=0
-  run "./mpiWorkThreadMult" $1
+  run "./mpiWorkThreadMult.${1}" $2
 }
 getAvg() {
-  awk '/realTime/ {sum+=$2; cnt+=1;} END {print "average " sum/cnt}' work${1}.log 
+  t=$(awk '/realTime/ {sum+=$2; cnt+=1;} END {print sum/cnt}' work${1}.log)
+  echo -n ", $t "
 }
 
 declare -a fns
 fns[1]="threadWorkComm"
 fns[2]="mpiWork"
 fns[3]="mpiWorkThreadMult"
 
-for workers in 2 4 8 16 32; do
-  for i in "${!fns[@]}"; do   
-    cat /dev/null > work${workers}.log
-    echo -n "${fns[i]} workers $workers "
-    for j in {0..4}; do 
-      ${fns[$i]} $workers
+for fnIdx in "${!fns[@]}"; do
+  for affinity in 1 2 4; do 
+    echo -n "${fns[$fnIdx]}, $affinity"
+    for workers in 2 4 8 16 32; do
+      cat /dev/null > work${workers}.log
+      for j in {0..4}; do 
+        ${fns[$fnIdx]} $affinity $workers
+      done
+      getAvg $workers
     done
-    getAvg $workers
+    echo ""
   done
 done

getTimeMvapich2.sh

@@ -1,30 +1,35 @@
 #!/bin/bash -e
 threadWorkComm() { 
-  workers=$1
+  affinity=$1
+  workers=$2
   echo "mic0:1" > ./hostsThreadWorkComm
   echo "export MV2_ENABLE_AFFINITY=0" > ./threadWorkCommMvapich2.sh
-  echo "$PWD/threadWorkComm $workers" >> ./threadWorkCommMvapich2.sh
+  echo "$PWD/threadWorkComm.${affinity} $workers" >> ./threadWorkCommMvapich2.sh
   chmod +x ./threadWorkCommMvapich2.sh
   echo "-n 1 : $PWD/threadWorkCommMvapich2.sh " > ./configThreadWorkComm
-  /usr/bin/time -f "realTime %e" mpirun_rsh \
+  mpirun_rsh \
     -config ./configThreadWorkComm \
-    -hostfile ./hostsThreadWorkComm &>> work${1}.log
+    -hostfile ./hostsThreadWorkComm &>> work${workers}.log
 }
 
 getAvg() {
-  awk '/realTime/ {sum+=$2; cnt+=1;} END {print "average " sum/cnt}' work${1}.log 
+  t=$(awk '/realTime/ {sum+=$2; cnt+=1;} END {print sum/cnt}' work${1}.log)
+  echo -n ", $t "
 }
 
 declare -a fns
 fns[1]="threadWorkComm"
 
-for workers in 2 4 8 16 32; do
-  for i in "${!fns[@]}"; do   
-    cat /dev/null > work${workers}.log
-    echo -n "${fns[i]} workers $workers "
-    for j in {0..4}; do 
-      ${fns[$i]} $workers
+for fnIdx in "${!fns[@]}"; do
+  for affinity in 1 2 4; do 
+    echo -n "${fns[$fnIdx]}, $affinity"
+    for workers in 2 4 8 16 32; do
+      cat /dev/null > work${workers}.log
+      for j in {0..4}; do 
+        ${fns[$fnIdx]} $affinity $workers
+      done
+      getAvg $workers
     done
-    getAvg $workers
+    echo ""
   done
 done

mpiThreadMultiplePerformanceOnPhi.png

@@ -10,6 +10,7 @@
 int main(int argc, char** argv) {
     int rank;
     int worldSz;
+    double t0;
 #ifdef THREAD_MULTIPLE
     int provided;
     MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
@@ -23,12 +24,15 @@ int main(int argc, char** argv) {
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &worldSz);
     thdata* data = (thdata*) calloc(1,sizeof(thdata));
+    t0 = MPI_Wtime();
     data->rank = rank;
     data->commsz = worldSz;
     data->id = rank;
     data->peers = 1;
     kernelComm((void*) data);
     MPI_Barrier(MPI_COMM_WORLD);
+    if( !rank )
+      fprintf(stderr, "realTime %.3f\n", MPI_Wtime()-t0);
     free(data);
     MPI_Finalize();
     return 0;

mpiWork.c

@@ -0,0 +1,29 @@
+   ,      ,         , workers
+mpi, model, threads-per-core, 2, 4, 8, 16, 32
+impi4, threadWorkComm, 1, 1.0942 , 2.6856 , 7.444 , 19.8354 , 44.3168 
+impi4, threadWorkComm, 2, 1.847 , 2.741 , 7.0572 , 17.7922 , 43.8934 
+impi4, threadWorkComm, 4, 1.5304 , 2.417 , 6.7752 , 16.9144 , 42.0378 
+impi4, mpiWork, 1, 0.0444 , 0.0436 , 0.0438 , 0.0436 , 0.0436 
+impi4, mpiWork, 2, 0.1454 , 0.1438 , 0.1444 , 0.144 , 0.144 
+impi4, mpiWork, 4, 0.251 , 0.2522 , 0.2532 , 0.2528 , 0.2526 
+impi4, mpiWorkThreadMult, 1, 0.0742 , 0.0734 , 0.074 , 0.0736 , 0.0738 
+impi4, mpiWorkThreadMult, 2, 0.1992 , 0.1994 , 0.1998 , 0.202 , 0.1988 
+impi4, mpiWorkThreadMult, 4, 0.3576 , 0.3556 , 0.3594 , 0.355 , 0.3578 
+impi5, threadWorkComm, 1, 0.7422 , 2.7226 , 7.1064 , 18.4822 , 42.765 
+impi5, threadWorkComm, 2, 0.9698 , 2.3846 , 6.4668 , 15.289 , 41.4386 
+impi5, threadWorkComm, 4, 0.9668 , 2.0686 , 6.0412 , 13.7852 , 38.2496 
+impi5, mpiWork, 1, 0.0226 , 0.023 , 0.023 , 0.0246 , 0.0254 
+impi5, mpiWork, 2, 0.0622 , 0.0632 , 0.0622 , 0.0622 , 0.063 
+impi5, mpiWork, 4, 0.1628 , 0.1592 , 0.156 , 0.157 , 0.1584 
+impi5, mpiWorkThreadMult, 1, 0.037 , 0.0362 , 0.0364 , 0.0376 , 0.0364 
+impi5, mpiWorkThreadMult, 2, 0.0902 , 0.0904 , 0.0886 , 0.0896 , 0.0894 
+impi5, mpiWorkThreadMult, 4, 0.234 , 0.2324 , 0.219 , 0.2206 , 0.2286 
+impi51, threadWorkComm, 1, 0.7728 , 2.807 , 6.9916 , 18.293 , 42.3842 
+impi51, threadWorkComm, 2, 1.4986 , 2.3072 , 6.3904 , 15.2192 , 41.3416 
+impi51, threadWorkComm, 4, 1.2304 , 2.1484 , 6.0166 , 13.7792 , 37.9134
+mvapich2-mic, threadWorkComm, 1, 38.3682 , 10.404 , 5.944 , 12.6848 , 28.5394 
+mvapich2-mic, threadWorkComm, 2, 147.384 , 5.735 , 5.466 , 11.84 , 24.4438 
+mvapich2-mic, threadWorkComm, 4, >100, >100, 7.9684 , 12.4788 , 25.419 
+mvapich2-mic2, threadWorkComm, 1, 1.1414 , 6.3 , 5.3218 , 12.2706 , 25.9254 
+mvapich2-mic2, threadWorkComm, 2, 0.6936 , 2.2848 , 4.9388 , 10.5948 , 22.7568 
+mvapich2-mic2, threadWorkComm, 4, 0.6854 , 2.7284 , 5.356 , 10.8154 , 23.2128 

performance.csv

@@ -42,8 +42,7 @@ hostnp=0 #tasks per host
 #micnp=240
 micnp=1 #tasks per accelerator
 n_mics=1 #accelerators per host
-app="threadWorkComm"
-args="16"
+app=$1
 
 #The rest of this script should be safe to ignore
 #(Give or take performance)
@@ -69,9 +68,9 @@ MIC_ENVS="-genv PATH $SINK_PATH -genv LD_LIBRARY_PATH $SINK_LD_LIBRARY_PATH"
 
 for h in "${hosts[@]}"; do
 	for i in `seq 0 $(( $n_mics-1 ))`; do
-		printf -- "$MIC_ENVS -n %d -host %s %s\n" $micnp "$h-mic$i" "$app $args" >> $tmpdir/mpi_conf
+		printf -- "$MIC_ENVS -n %d -host %s %s\n" $micnp "$h-mic$i" "$app" >> $tmpdir/mpi_conf
 	done
-		printf -- "$HOST_ENVS -n %d -host %s %s\n" $hostnp "$h-br0" "$app $args" >> $tmpdir/mpi_conf
+		printf -- "$HOST_ENVS -n %d -host %s %s\n" $hostnp "$h-br0" "$app" >> $tmpdir/mpi_conf
 done
 
 cat $tmpdir/mpi_conf