cleaning.py

en_stopwords = list(set(nltk.corpus.stopwords.words('english')))

# remove punctuation from data
clean = [re.sub(r'[^\w\s]','',i).lower() for i in data]

tokens = [word_tokenize(x) for x in data['text']]
filtered_tokens = []

# tokens that are not stopwords collected here
for i in tokens:
  filtered_tokens.append([])
  for j in i:
    if j in en_stopwords:
      continue
    else: filtered_tokens[-1].append(j)
      
# initialize Lancaster Stemmer
LS = LancasterStemmer()
lemmatized = []
for l in filtered_tokens: lemmatized.append([LS.stem(w) for w in l])