Run OSU Benchmark on AWS ParallelCluster

run_osu_on_aws_parallelcluster.md

Benchmark OSU

prepare OSU micro benchmarks

wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.6.2.tar.gz
tar xvf osu-micro-benchmarks-5.6.2.tar.gz

run with OpenMPI

build

module load openmpi
cd osu-micro-benchmarks-5.6.2
./configure CC=/opt/amazon/openmpi/bin/mpicc CXX=/opt/amazon/openmpi/bin/mpicxx
make

submit job

• osu.sh

#!/bin/bash
#$ -cwd
#$ -pe mpi 72

BIN_PATH=/home/centos/osu-micro-benchmarks-5.6.2/mpi/pt2pt
mpiexec --map-by node -np 2 ${BIN_PATH}/osu_bw
mpiexec --map-by node -np 2 ${BIN_PATH}/osu_latency

$ qsub osu.sh

run with Intel MPI

build

module load intelmpi
cd osu-micro-benchmarks-5.6.2
./configure CC=/opt/intel/compilers_and_libraries_2020.0.166/linux/mpi/intel64/bin/mpicc CXX=/opt/intel/compilers_and_libraries_2020.0.166/linux/mpi/intel64/bin/mpicxx
make

submit job

• osu-intel.sh

#!/bin/bash
#$ -cwd
#$ -pe mpi 72

INTEL_OPT="-print-rank-map -perhost 1 -n 2"
OSU_PATH=/home/centos/osu-micro-benchmarks-5.6.2/mpi/pt2pt

module load intelmpi
#export FI_PROVIDER=sockets
export FI_PROVIDER=efa

mpiexec ${INTEL_OPT} ${OSU_PATH}/osu_bw
mpiexec ${INTEL_OPT} ${OSU_PATH}/osu_latency

$ qsub osu-intel.sh

Results

# OSU MPI Bandwidth Test v5.6.2
# Size      Bandwidth (MB/s)
1                       0.59
2                       1.19
4                       2.42
8                       1.70
16                      9.63
32                     19.25
64                     38.46
128                    78.35
256                   161.59
512                   311.80
1024                  610.79
2048                 1158.92
4096                 2298.75
8192                 4146.65
16384                 869.22
32768                6118.97
65536                7103.44
131072               8119.87
262144               8878.30
524288               9292.81
1048576              8646.58
2097152              8180.50
4194304              7822.21
# OSU MPI Latency Test v5.6.2
# Size          Latency (us)
0                      17.99
1                      19.78
2                      17.63
4                      17.40
8                      17.34
16                     17.36
32                     17.38
64                     17.42
128                    17.52
256                    17.58
512                    17.93
1024                   18.70
2048                   20.39
4096                   23.14
8192                   27.98
16384                  71.20
32768                  77.08
65536                  85.02
131072                 98.69
262144                138.94
524288                247.28
1048576               451.81
2097152               898.88
4194304              1738.13