Example Wordcount in Spark

spark_wordcount.py

from pyspark import SparkContext
from pyspark import SparkConf

if __name__ == "__main__":
    conf = SparkConf()
    conf.setMaster("spark://{hostname}:7077")
    conf.setAppName("WordCount")
    sc = SparkContext(conf=conf)

file = sc.textFile("/mnt/gluster/pg2591.txt")
counts = file.map(lambda word: word.replace('"', '').replace("'", '').replace('.', ' ').replace(',', ' ').replace(':', '').replace(';', ' ').lower()) \
    .flatMap(lambda line: line.split()) \
    .map(lambda word: (word, 1)) \
    .reduceByKey(lambda a, b: a + b) \
    .map(lambda x: (x[1], x[0])) \
    .sortByKey(False)
counts.saveAsTextFile("/mnt/gluster/wordcount")