MrJobを使ったMapReduce処理記述と実行

mapper.py

#! /usr/bin/env python
# -*- coding: utf-8 -*- 

from mrjob.job import MRJob

class MRWordCounter(MRJob):
    def mapper(self, key, line):
        for word in line.split():
            yield word, 1

    def reducer(self, word, occurrences):
        yield word, sum(occurrences)

if __name__ == '__main__':
    MRWordCounter.run()

mrjob_local.sh

#ローカルでの実行
python wc.py < creativecommons.txt

mrjob_s3.sh

#s3上ファイルを使ってEMR上で実行


export AWS_ACCESS_KEY_ID=<your aws access key>
export AWS_SECRET_ACCESS_KEY=<your secret access key>

python wc.py -r emr s3://masayang-bootcamp/bootcamp4/EMRconsole/creativecommons.txt s3://masayang-bootcamp/bootcamp4/EMRconsole/creativecommons.txt -o s3://masayang-bootcamp/bootcamp4/EMRconsole/<your account>