spam

spam_003.py

print("removing punctuations: "+string.punctuation)
stemmer = PorterStemmer()

def tokenize(message):
    """ removes punctuation and tokenizes the words and stems each word.
    """
    msg = "".join([ch for ch in message if ch not in string.punctuation]) # get rid of punctuations
    tokens = word_tokenize(msg) 
    #stems = [stemmer.stem(x).lower() for x in tokens] #correct way to do
    stems = [x.lower() for x in tokens] #iOS does not have porterstemmer, we are going to not use stem for now
    return stems

messages.message.head().apply(tokenize)