re_nlp.py

import re
import nltk
nltk.download('stopwords')

# download stopwords list from nltk
from nltk.corpus import stopwords

stop_words = set(stopwords.words('english'))

def clean_text(text):
    # converting to lowercase
    newString = text.lower()
    # removing links
    newString = re.sub(r'(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b', '', newString) 
    # fetching alphabetic characters
    newString = re.sub("[^a-zA-Z]", " ", newString)
    # removing stop words
    tokens = [w for w in newString.split() if not w in stop_words]
    # removing short words
    long_words=[]
    for i in tokens:
        if len(i)>=4:                                                 
            long_words.append(i)   
    return (" ".join(long_words)).strip()