Hadoop Streaming

example_streaming.sh

#!/bin/bash
hadoop="/usr/bin/hadoop"
STREAMING=/opt/cloudera/parcels/CDH/lib/hadoop-mapreduce/hadoop-streaming.jar
cwd=`pwd`

mapper="${cwd}/getcvlist.py"
reducer="${cwd}/getcvlist.py"

echo $1 $2

uuidfile=$2

inputdir=""
inputdir="${inputdir} -input /path/to/input "

outputdir="/user/ktaguchi/test"

${hadoop} fs -rm -r ${outputdir}

${hadoop} jar ${STREAMING} \
      -Dmapred.output.compress=true \
      -Dmapred.output.compression.codec=org.apache.hadoop.io.compress.DefaultCodec \
      -Dmapred.reduce.tasks=0 \
      -mapper "cut -f 2-6" \
      ${inputdir} \
      -output ${outputdir} \
      -inputformat 'org.apache.hadoop.mapred.SequenceFileAsTextInputFormat' \
      -outputformat 'org.apache.hadoop.mapred.SequenceFileOutputFormat'

#${hadoop} fs -text "${outputdir}/part*" > ./results/uu_list_data_1