Shows how to filter entries out based on some logic in Spark

filterout.py

#reading data from a file
logData = sc.textFile(logFile)

X = 10

#for each key finding entries that occur more than X times
outliers = logData.map(lambda (k, v): (k, 1)).reduceByKey(lambda a, b: a + b).filter(lambda (k, v): v > X).cache()

#filtering these entries out
reducedLogData = logData.subtractByKey(outliers).cache()