classifier_initialization.py

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction import stop_words
from nltk.corpus import stopwords

# initializing the stop-words list
my_stopwords = stopwords.words('english')

# adding words that are in the vectorizer
# but are not actual words (unicode characters)
my_stopwords.extend(['amp','x200b','\n'])

# replacing the target values with 1s and 0s
# 1 for 'shower_thoughts' and 0 for 'deep_philosophy'
df['subreddit'].replace({'shower_thoughts': 1, 'deep_philosophy': 0}, inplace = True)

# defining X and y
X = df['post']
y = df['subreddit']

# 80/20 train/test split
X_train, X_test, y_train, y_test = train_test_split(X, y, stratify = y, test_size = 0.2)

# instatiating the TfidfVectorizer
tfidvec = TfidfVectorizer(stop_words = my_stopwords)

# getting the sparse matrices for X_train and X_test
tfidvec.fit(X_train)
X_train_tfidf = tfidvec.transform(X_train)
X_test_tfidf = tfidvec.transform(X_test)