JeremiahKamama

tfvectorizer = TfidfVectorizer(tokenizer = my_tokenizer)

text_cf = Pipeline([('tfidf', TfidfVectorizer()),('classifier',LinearSVC(),)])

JeremiahKamama

train_pad = pad_sequences(train_tokens, maxlen=max_tokens)
test_pad = pad_sequences(test_tokens, maxlen=max_tokens)

JeremiahKamama

num_tokens = [len(tokens) for tokens in train_tokens + test_tokens]
num_tokens = np.array(num_tokens)

JeremiahKamama

plt.figure(figsize = (20,20))
cloud = WordCloud(max_words= 2000, width= 1600,height= 800).generate("".join(imdb[imdb.sentiment == 0].review))
plt.imshow(cloud, interpolation='bilinear')

JeremiahKamama

max_tokens = np.mean(num_tokens) + 2 * np.std(num_tokens)
max_tokens = int(max_tokens)
max_tokens

JeremiahKamama

embedding_size=100

model = Sequential()

model.add(Embedding(input_dim=15000,output_dim=embedding_size,input_length=max_tokens,name="embedding_layer"))

model.add(LSTM(units=16,return_sequences= True))
model.add(Dropout(0.1))

model.add(LSTM(units=8,return_sequences=True))

JeremiahKamama

plt.figure()
plt.plot(history.history["loss"], label = "Train")
plt.plot(history.history["val_loss"], label = "Test")
plt.title("Loss")
plt.ylabel("Acc")
plt.xlabel("epochs")
plt.legend()
plt.show()

JeremiahKamama

idx = tokenizer.word_index
inverse_map = dict(zip(idx.values(), idx.keys()))

def return_sentence(tokens):
    words = [inverse_map[token] for token in tokens if token!=0]
    text = ' '.join(words)
    return text

JeremiahKamama

from tensorflow.python.keras.preprocessing.text import Tokenizer
from keras.preprocessing import sequence
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Embedding,LSTM
from tensorflow.python.keras.preprocessing.text import Tokenizer
from keras.optimizers import Adam
from keras.layers import Dropout
from tensorflow.python.keras.preprocessing.sequence import pad_sequences
from tensorflow.python.keras.models import load_model

JeremiahKamama

import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import numpy as np
import spacy
import string
import xgboost as xgb
import re