nithyadurai87

import numpy as np
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, LSTM, Dense
from tensorflow.keras.preprocessing.sequence import pad_sequences
import pickle

files = [r'/content/இசை_ஜீனியஸ்_ராஜா_ரவி_நடராஜன்.txt',r'/content/தமிழின்_எதிர்காலமும்_தகவல்_தொழில்நுட்பமும்_இரா_அசோகன்.txt',r'/content/திறந்த_மூல_மென்பொருளில்_முதல்_அடி_எடுத்து_வைக்கலாம்_வாங்க_இரா_அசோகன்.txt',r'/content/தொழிலியல்_விஞ்ஞானி_ஜி_டி_நாயுடு_என்_வி_கலைமணி.txt',r'/content/நான்_இந்துவல்ல_நீங்கள்_தொ_பரமசிவம்.txt']

nithyadurai87

from tensorflow.keras.models import model_from_json
from tensorflow.keras.preprocessing.sequence import pad_sequences
import pickle

tokens = pickle.load(open(r'/content/Ilayaraja_book_tokens.pkl', 'rb'))
model_file = pickle.load(open(r'/content/Ilayaraja_book_model.pkl', 'rb'))

model = model_from_json(model_file['model_json'])
model.set_weights(model_file['model_weights'])
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

nithyadurai87

import numpy as np
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, LSTM, Dense
from tensorflow.keras.preprocessing.sequence import pad_sequences
import pickle

x = open(r'/content/இசை_ஜீனியஸ்_ராஜா_ரவி_நடராஜன்.txt', 'rb').read().decode(encoding='utf-8')
x = x.replace('\n', '').replace('\r', '').replace('\ufeff', '').replace('“','').replace('”','')

nithyadurai87

from gensim.models.fasttext import FastText 

paragraph = "Periyar was a social reformer in Tamil Nadu. He founded the Self-Respect Movement. This movement aimed to promote equality and end caste discrimination. Today, he is celebrated as a key figure in the fight for social justice and equality in Tamil Nadu"
x = [i for i in paragraph.split('.')]
x1= [[word for word in nltk.word_tokenize(sentence) if word.lower() not in nltk.corpus.stopwords.words('english')] for sentence in x]

model = FastText(x1, window=20, min_count=1, sg=1, sample=1e-3)

print (model.wv.index_to_key)
print (model.wv['Periyar'])

nithyadurai87

from gensim.models import word2vec  

paragraph = "Periyar was a social reformer in Tamil Nadu. He founded the Self-Respect Movement. This movement aimed to promote equality and end caste discrimination. Today, he is celebrated as a key figure in the fight for social justice and equality in Tamil Nadu"
x = [i for i in paragraph.split('.')]
x1= [[word for word in nltk.word_tokenize(sentence) if word.lower() not in nltk.corpus.stopwords.words('english')] for sentence in x]

model = word2vec.Word2Vec(x1, window=10, vector_size=5, min_count=1, sg=1, sample=1e-3)

print (model.wv.index_to_key)
print (model.wv['Periyar'])

nithyadurai87

from sklearn.preprocessing import LabelEncoder
import numpy as np

paragraph = "Periyar was a social reformer in Tamil Nadu. He founded the Self-Respect Movement. This movement aimed to promote equality and end caste discrimination. Today, he is celebrated as a key figure in the fight for social justice and equality in Tamil Nadu."
x = [i for i in paragraph.split('.')]

l1 = []
for i in x:
  l1.append(LabelEncoder().fit_transform(i.split()))
padded_arrays = [np.pad(i, (0, max(len(i) for i in l1) - len(i)), 'constant', constant_values=99) for i in l1]

nithyadurai87

import nltk
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('punkt_tab')   
from sklearn.feature_extraction.text import CountVectorizer

paragraph = "Periyar was a social reformer in Tamil Nadu. He founded the Self-Respect Movement. This movement aimed to promote equality and end caste discrimination. Today, he is celebrated as a key figure in the fight for social justice and equality in Tamil Nadu."
x = [i for i in paragraph.split('.')]

tokens = CountVectorizer()

nithyadurai87

import numpy as np
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, LSTM, Dense
from tensorflow.keras.preprocessing.sequence import pad_sequences

x = "தமிழ்நாடு இந்தியாவின் தெற்கே அமைந்த ஒரு அழகிய மாநிலமாகும். இது பல்வேறு கலாச்சார பாரம்பரியங்களையும், செழிப்பான சாகுபடிமுறையையும் கொண்டுள்ளது. தமிழ்நாட்டின் தலைநகரமான சென்னை, தொழில்நுட்பம் மற்றும் கல்வியில் முன்னணி வகிக்கிறது. மாமல்லபுரம், தஞ்சாவூர் பெரிய கோயில் போன்ற வரலாற்று முக்கியத்துவம் வாய்ந்த இடங்கள் சுற்றுலாப் பயணிகளை ஈர்க்கின்றன. தமிழ்நாட்டின் கலை, இலக்கியம் மற்றும் இசை உலகளாவிய புகழ் பெற்றவை"

tokens = Tokenizer()

nithyadurai87

import pandas as pd
from datetime import datetime,timedelta
import numpy as np

df = pd.read_csv('./13_input_data.csv')
print (df)

pd.set_option("display.max_columns",8) 

df1 = pd.DataFrame()

nithyadurai87

import pandas as pd

d = {'Names' : pd.Series(['Mahesh','Yazhini','Kadiresan','Malathi','Kumar','Sujith']), 
'Gender' : pd.Series(['Male','Trans','Male','Female','Male','Trans'],dtype="category")}
df = pd.DataFrame(d)

print (df['Names'])
print (df['Gender'])
print (df['Gender'].cat.remove_categories(['Trans'])) # add_categories()
print (df['Gender'].cat.categories)