santhalakshminarayana

tot_pairs = len(co_occ_matrix)
p1, p2,occs = list(), list(), list()
for i in co_occ_matrix.keys():
    p1.append(i[0])
    p2.append(i[1])
    occs.append(co_occ_matrix[i])
p1 = np.array(p1)
p2 = np.array(p2)
occs = np.array(occs)

santhalakshminarayana

co_occ_matrix = defaultdict(int)
window = 5 # not greater than 5
for sent in quotes:
    words = sent.split(' ')
    # first window 
    for i in range(0, window):
        for j in range(i+1, window):
            weight = 1/(j-i)
            ind_1 = word_to_int[words[i]]
            ind_2 = word_to_int[words[j]]

santhalakshminarayana

words_dict = dict()
def unique_words(data_list):
    for x in data_list:
        for word in x.split(' '):
            words_dict[word] = 1
    return words_dict.keys()
  
words = list(unique_words(quotes))
words.sort()

santhalakshminarayana

quotes = []
max_len = 0
min_len = 5
sent_len_dic = defaultdict(int)
with open('quotes.txt', 'r') as f:
  while True:
    quote = f.readline()
    if not quote:
      break
    words = quote.split(' ')

santhalakshminarayana

def get_quotes(url):
    i = 1
    quotes = []
    while True:
        curr_quotes = []
        quote_url = url + 'page-' + str(i) + '/'
        i += 1
        quote_r = requests.get(quote_url)
        quote_soup = BeautifulSoup(quote_r.content, 'html5lib')
        quote_list = quote_soup.find('div', attrs = {'class':'quote_list'})

santhalakshminarayana

import requests
from bs4 import BeautifulSoup
import re
import os
import time

home_url = 'http://www.wiseoldsayings.com'
r = requests.get(home_url)
soup = BeautifulSoup(r.content, 'html5lib')
quote_classes = []

santhalakshminarayana

text_count = 0
input_text = 'PRIYA'
while text_count < 50:
  gen_text = generate(input_text)
  gen_text = gen_text.strip('.')
  if len(gen_text) > 6 and gen_text not in names:
    text_count += 1
    input_text = gen_text
    print(gen_text)

santhalakshminarayana

def generate(base_name):
  if len(base_name):
    base_name = base_name[:window]
  x = np.zeros((1,window,len(int_to_char)))
  seq_word = []
  ind_list = [char_to_int[i] for i in base_name]
  for i,ind in enumerate(ind_list):
    x[0 ,i ,ind] = 1
    seq_word.append(ind)

santhalakshminarayana

model = Sequential()
model.add(LSTM(units=32, recurrent_dropout=0.5, input_shape=(window, len(int_to_char))))
model.add(Dense(len(int_to_char), activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer='adam')
model.fit(X, Y, batch_size=1024, epochs=10 ,steps_per_epoch=3000)

santhalakshminarayana

X = np.zeros(shape = (len(sequences), window, len(int_to_char)))
Y = np.zeros(shape = (len(next_chars),len(int_to_char)))
for i in range(len(sequences)):
  for j in range(window):
    X[i, j, char_to_int[sequences[i][j]]] = 1
  Y[i, char_to_int[next_chars[i]]] = 1