Script to test wall time of teragen/terasort exercise

TeraTune.sh

#!/bin/sh

## Test for optimal mix of mappers, reducers, and container allocations

## Modified from ripped-off script, attributed to BVT and excised Cloudera docs

## Paths below correspond to active parcel installation
MR=/opt/cloudera/parcels/CDH/lib/hadoop-0.20-mapreduce
CDH=/opt/cloudera/parcels/CDH/bin

## 51,200,000 records @ 100 bytes/record = 5.12 GB
RECORDS=51200000

# Mark start of the loop
echo Testing loop started on `date`

# Mapper tasks
# Set this value to some multiple of worker nodes
for maps in 8
do
   # Reducer tasks
   # Set this value to some factor of worker nodes
   for reds in 1
   do
      # Container memory
      for mem in 512 1024
      do
         # Set JVM heap
         mheap=`echo "(${mem}*0.8)/1" | bc`
         rheap=`echo "(${mem}*0.8)/1" | bc`

         time ${CDH}/hadoop jar ${MR}/hadoop-examples.jar teragen \
                     -Dmapreduce.job.maps=${maps} \
                     -Dmapreduce.map.memory.mb=${mem} \
                     -Dmapreduce.map.java.opts.max.mem=${mheap} \
                     ${RECORDS} /results/tg-10GB-${maps}-${mem} \
                     1>tera_${maps}_${mem}.out \
                     2>tera_${maps}_${mem}.err

        time ${CDH}/hadoop jar ${MR}/hadoop-examples.jar terasort \
                     -Dmapreduce.job.maps=${maps} \
                     -Dmapreduce.job.reduces=${reds} \
                     -Dmapreduce.map.memory.mb=${mem} \
                     -Dmapreduce.map.java.opts.max.mem=${mheap} \
                     -Dmapreduce.reduce.memory.mb=${mem} \
                     -Dmapreduce.reduce.java.opts.max.mem=${rheap} \
                     /results/tg-10GB-${maps}-${mem}  \
                     /results/ts-10GB-${maps}-${reds}-${mem} \
                     1>>tera_${maps}_${reds}_${mem}.out \
                     2>>tera_${maps}_${reds}_${mem}.err

        $HADOOP/hadoop fs -rm -r -skipTrash /results/tg-10GB-${maps}-${mem}
        $HADOOP/hadoop fs -rm -r -skipTrash /results/ts-10GB-${maps}-${red}-${mem}
      done
   done
done

echo Testing loop ended on `date`