text_processing.py

import re
def processTweet(chat):
    chat = chat.lower()
    chat = re.sub('((www\.[^\s]+)|(https?://[^\s]+))','',chat)
    chat = re.sub('@[^\s]+','',chat)
    chat = re.sub('[\s]+', ' ', chat)
    chat = re.sub(r'#([^\s]+)', r'\1', chat)
    chat = re.sub(r'[\.!:\?\-\'\"\\/]', r'', chat)
    chat = chat.strip('\'"')
    return chat

def replaceTwoOrMore(s):
    #look for 2 or more repetitions of character and replace with the character itself
    pattern = re.compile(r"(.)\1{2,}", re.DOTALL)
    return pattern.sub(r"\1", s)

def getFeatureVector(chat):
    chat=processTweet(chat)
    featureVector = []
    #split tweet into words
    words = chat.split()
    for w in words:
        #replace two or more with two occurrences
        w = replaceTwoOrMore(w)
        #strip punctuation
        w = w.strip('\'"?,.')
        #check if the word stats with an alphabet
        val = re.search(r"^[a-zA-Z][a-zA-Z0-9]*$", w)
        #ignore if it is a stop word
        if(val is None):
            continue
        else:
            featureVector.append(w.lower())
    return " ".join(list(featureVector))