emmagrimaldi

# importing the libraries needed in this section
import nltk
from nltk.tokenize import RegexpTokenizer

# defining the text
text = 'Rome was founded in 753BC by its first king, Romulus.'

# instantiating the tokenizer object. By passing r'\w+' to the RegexpTokenizer
# I am selecting groups of single words, discarding the punctuation
tokenizer = RegexpTokenizer(r'\w+')

emmagrimaldi

# importing the library needed in this section
from nltk.corpus import stopwords

# assigning the english stop-words to the sw list
sw = stopwords.words('english')

# assigning the non stop-words contained in the tokens list
# to a new list named clean_tokens through a list comprehension
clean_tokens = [token for token in tokens if token not in sw]

emmagrimaldi

# importing the library needed in this section
from nltk.stem import WordNetLemmatizer

# instantiating the lemmaztizer object
lemmatizer = WordNetLemmatizer()

# lemmatizing each word through a list comprehension
[lemmatizer.lemmatize(token) for token in clean_tokens]

emmagrimaldi

# importing the library needed in this section
from nltk.stem.porter import PorterStemmer

# instantiating the stemmer object
pstemmer = PorterStemmer()

# stemming each word through a list comprehension
[pstemmer.stem(token) for token in clean_tokens]

emmagrimaldi

from sklearn.tree import DecisionTreeClassifier
import pandas as pd

# create a matrix including the data
data = [[8,8,'dog'],[50,40,'dog'],[8,9,'cat'],[15,12,'dog'],[9,9.8,'cat']]

# generating a dataframe from the matrix 
df = pd.DataFrame(data, columns = ['weight','height','label'])

# defining predictors

emmagrimaldi

from sklearn.externals.six import StringIO  
from sklearn.tree import export_graphviz
import pydotplus
from IPython.display import Image


dot_data = StringIO()  

export_graphviz(
    model, 

emmagrimaldi

import requests
import json
import time
import pandas as pd

# changing the user-agent 
headers = {'User-agent': 'E Bot 1.0'}

# setting after = None so I can start from the first page
after = None

emmagrimaldi

# the range of the iteration starts from 2 because the first 2 
# elements are not posts but instructions related to the page
shower_thoughts_list = [shower_thoughts[i]['data']['title'] + ' ' +
                        shower_thoughts[i]['data']['selftext'] 
                        for i in range(2,len(shower_thoughts))]

# creating the dataframe from the list
st_df = pd.DataFrame(shower_thoughts_list, columns=['post'])

# adding the subreddit column

emmagrimaldi

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction import stop_words
from nltk.corpus import stopwords

# initializing the stop-words list
my_stopwords = stopwords.words('english')

# adding words that are in the vectorizer
# but are not actual words (unicode characters)
my_stopwords.extend(['amp','x200b','\n'])

emmagrimaldi

from sklearn.metrics import confusion_matrix

def conf_matrix(model, X_test):
    y_pred = model.predict(X_test)            # calculate predictions
    cm = confusion_matrix(y_test, y_pred)     # defining the confusion matrix
    tn, fp, fn, tp = cm.ravel()               # assigning the elements of the confusion matrix to variables
    print(f"True Negatives: {tn}")            # print those variables
    print(f"False Positives: {fp}")
    print(f"False Negatives: {fn}")
    print(f"True Positives: {tp}")            # return the confusion matrix as dataframe