Haderp

gistfile1.sh

#!/bin/bash

export PATH=$PATH:/opt/hadoop/hadoop-0.20.2/bin/

HDFS_PATH="/user/s1317371"

if [ -e $1 ]; then
    rm $1
fi

for i in {1..8}; do
cat >> $1 <<EOF
Task $i code begin
Task $i mapper begin
$(<$i/mapper.py)
Task $i mapper end
Task $i reducer begin
$(<$i/reducer.py)
Task $i reducer end
Task $i run script begin
$(<$i/run)
Task $i run script end
Task $i code end
Task $i results begin
$(if [ $i = 3 -o $i = 4 ]; then
    hadoop dfs -cat "$HDFS_PATH/s1317371_task_$i.out/result.txt" 2>/dev/null
    sleep 1
else
    hadoop dfs -cat "$HDFS_PATH/s1317371_task_$i.out/part-00000" 2>/dev/null | head -n 20
fi)
Task $i results end
EOF
done

# Task 1 has no reducer
sed -i "/Task 1 reducer/d" $1