Skip to content

Instantly share code, notes, and snippets.

@marty1885

marty1885/Esperanto-Keras-RNN.ipynb

Created November 27, 2018 09:31
Show Gist options
  • Save marty1885/3695fe91b7a42c601dd87a9ad51af2d5 to your computer and use it in GitHub Desktop.
Save marty1885/3695fe91b7a42c601dd87a9ad51af2d5 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
Esperanto-Keras-RNN.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using MXNet backend\n"
]
}
],
"source": [
"# Larger LSTM Network to Generate Text for Alice in Wonderland\n",
"import numpy\n",
"import numpy as np\n",
"from keras.models import Sequential\n",
"from keras.layers import Dense, TimeDistributed\n",
"from keras.layers import Dropout, Reshape\n",
"from keras.layers import LSTM, GRU\n",
"from keras.callbacks import ModelCheckpoint\n",
"from keras.utils import np_utils"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"esperanto_cap_alphabet = ['A', 'B', 'C', 'Ĉ', 'D', 'E', 'F', 'G', 'Ĝ', 'H', 'Ĥ', 'I', 'J',\n",
" 'Ĵ', 'K', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'Ŝ', 'T', 'U', 'Ŭ', 'V', 'Z', ' ']\n",
"esperanto_low_alphabet = ['a', 'b', 'c', 'ĉ', 'd', 'e', 'f', 'g', 'ĝ', 'h', 'ĥ', 'i', 'j',\n",
" 'ĵ', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's', 'ŝ', 't', 'u', 'ŭ', 'v', 'z', ' ']\n",
"\n",
"def esperantoToLower(ch):\n",
" if ch in esperanto_cap_alphabet:\n",
" return esperanto_low_alphabet[esperanto_cap_alphabet.index(ch)]\n",
" return ch\n",
"\n",
"def charToToken(ch):\n",
" c = esperantoToLower(ch)\n",
" base_len = len(esperanto_low_alphabet)+1\n",
" additional_chars = [' ']\n",
" if c == '.' or c == '?':\n",
" return 0 #. is a special token\n",
" elif c in esperanto_low_alphabet:\n",
" return esperanto_low_alphabet.index(c) + 1\n",
" return 0\n",
"\n",
"def encodeEsp(s):\n",
" tokens = []\n",
" for ch in s:\n",
" token = charToToken(ch)\n",
" if token != None:\n",
" one_hot = np.zeros(len(esperanto_cap_alphabet)+1)\n",
" one_hot[token] = 1\n",
" tokens += [one_hot]\n",
" return np.array(tokens)\n",
"\n",
"def decodeEsperanto(token):\n",
" if token == 0:\n",
" return '0'\n",
" return esperanto_low_alphabet[token-1]"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Total Characters: 2068\n",
"Total Vocab: 36\n",
"Total Patterns: 1968\n",
"(1968, 100, 30)\n",
"(1968, 30)\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"gru_15 (GRU) (None, 100, 256) 220416 \n",
"_________________________________________________________________\n",
"dropout_16 (Dropout) (None, 100, 256) 0 \n",
"_________________________________________________________________\n",
"gru_16 (GRU) (None, 256) 393984 \n",
"_________________________________________________________________\n",
"dropout_17 (Dropout) (None, 256) 0 \n",
"_________________________________________________________________\n",
"dense_8 (Dense) (None, 30) 7710 \n",
"=================================================================\n",
"Total params: 622,110\n",
"Trainable params: 622,110\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n",
"Epoch 1/50\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/marty/.local/lib/python3.7/site-packages/mxnet/module/bucketing_module.py:408: UserWarning: Optimizer created manually outside Module but rescale_grad is not normalized to 1.0/batch_size/num_workers (1.0 vs. 0.015625). Is this intended?\n",
" force_init=force_init)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"1968/1968 [==============================] - 12s 6ms/step - loss: 3.4242\n",
"\n",
"Epoch 00001: loss improved from inf to 3.42417, saving model to weights-improvement-01-3.4242.hdf5\n",
"Epoch 2/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 3.2522\n",
"\n",
"Epoch 00002: loss improved from 3.42417 to 3.25215, saving model to weights-improvement-02-3.2522.hdf5\n",
"Epoch 3/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 3.0356\n",
"\n",
"Epoch 00003: loss improved from 3.25215 to 3.03558, saving model to weights-improvement-03-3.0356.hdf5\n",
"Epoch 4/50\n",
"1968/1968 [==============================] - 14s 7ms/step - loss: 2.9184\n",
"\n",
"Epoch 00004: loss improved from 3.03558 to 2.91842, saving model to weights-improvement-04-2.9184.hdf5\n",
"Epoch 5/50\n",
"1968/1968 [==============================] - 13s 7ms/step - loss: 2.9045\n",
"\n",
"Epoch 00005: loss improved from 2.91842 to 2.90446, saving model to weights-improvement-05-2.9045.hdf5\n",
"Epoch 6/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9045\n",
"\n",
"Epoch 00006: loss did not improve from 2.90446\n",
"Epoch 7/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9026\n",
"\n",
"Epoch 00007: loss improved from 2.90446 to 2.90263, saving model to weights-improvement-07-2.9026.hdf5\n",
"Epoch 8/50\n",
"1968/1968 [==============================] - 13s 6ms/step - loss: 2.9019\n",
"\n",
"Epoch 00008: loss improved from 2.90263 to 2.90190, saving model to weights-improvement-08-2.9019.hdf5\n",
"Epoch 9/50\n",
"1968/1968 [==============================] - 13s 7ms/step - loss: 2.9026\n",
"\n",
"Epoch 00009: loss did not improve from 2.90190\n",
"Epoch 10/50\n",
"1968/1968 [==============================] - 13s 7ms/step - loss: 2.9039\n",
"\n",
"Epoch 00010: loss did not improve from 2.90190\n",
"Epoch 11/50\n",
"1968/1968 [==============================] - 13s 7ms/step - loss: 2.9030\n",
"\n",
"Epoch 00011: loss did not improve from 2.90190\n",
"Epoch 12/50\n",
"1968/1968 [==============================] - 13s 7ms/step - loss: 2.9027\n",
"\n",
"Epoch 00012: loss did not improve from 2.90190\n",
"Epoch 13/50\n",
"1968/1968 [==============================] - 13s 7ms/step - loss: 2.9030\n",
"\n",
"Epoch 00013: loss did not improve from 2.90190\n",
"Epoch 14/50\n",
"1968/1968 [==============================] - 13s 7ms/step - loss: 2.9024\n",
"\n",
"Epoch 00014: loss did not improve from 2.90190\n",
"Epoch 15/50\n",
"1968/1968 [==============================] - 13s 6ms/step - loss: 2.9017\n",
"\n",
"Epoch 00015: loss improved from 2.90190 to 2.90166, saving model to weights-improvement-15-2.9017.hdf5\n",
"Epoch 16/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9023\n",
"\n",
"Epoch 00016: loss did not improve from 2.90166\n",
"Epoch 17/50\n",
"1968/1968 [==============================] - 13s 6ms/step - loss: 2.9014\n",
"\n",
"Epoch 00017: loss improved from 2.90166 to 2.90140, saving model to weights-improvement-17-2.9014.hdf5\n",
"Epoch 18/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9046\n",
"\n",
"Epoch 00018: loss did not improve from 2.90140\n",
"Epoch 19/50\n",
"1968/1968 [==============================] - 13s 6ms/step - loss: 2.9003\n",
"\n",
"Epoch 00019: loss improved from 2.90140 to 2.90026, saving model to weights-improvement-19-2.9003.hdf5\n",
"Epoch 20/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9027\n",
"\n",
"Epoch 00020: loss did not improve from 2.90026\n",
"Epoch 21/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9008\n",
"\n",
"Epoch 00021: loss did not improve from 2.90026\n",
"Epoch 22/50\n",
"1968/1968 [==============================] - 13s 6ms/step - loss: 2.9012\n",
"\n",
"Epoch 00022: loss did not improve from 2.90026\n",
"Epoch 23/50\n",
"1968/1968 [==============================] - 14s 7ms/step - loss: 2.9030\n",
"\n",
"Epoch 00023: loss did not improve from 2.90026\n",
"Epoch 24/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9024\n",
"\n",
"Epoch 00024: loss did not improve from 2.90026\n",
"Epoch 25/50\n",
"1968/1968 [==============================] - 13s 6ms/step - loss: 2.9027\n",
"\n",
"Epoch 00025: loss did not improve from 2.90026\n",
"Epoch 26/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9021\n",
"\n",
"Epoch 00026: loss did not improve from 2.90026\n",
"Epoch 27/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9021\n",
"\n",
"Epoch 00027: loss did not improve from 2.90026\n",
"Epoch 28/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9035\n",
"\n",
"Epoch 00028: loss did not improve from 2.90026\n",
"Epoch 29/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9017\n",
"\n",
"Epoch 00029: loss did not improve from 2.90026\n",
"Epoch 30/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.8993\n",
"\n",
"Epoch 00030: loss improved from 2.90026 to 2.89930, saving model to weights-improvement-30-2.8993.hdf5\n",
"Epoch 31/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.8986\n",
"\n",
"Epoch 00031: loss improved from 2.89930 to 2.89863, saving model to weights-improvement-31-2.8986.hdf5\n",
"Epoch 32/50\n",
"1968/1968 [==============================] - 13s 6ms/step - loss: 2.9024\n",
"\n",
"Epoch 00032: loss did not improve from 2.89863\n",
"Epoch 33/50\n",
"1968/1968 [==============================] - 13s 6ms/step - loss: 2.9004\n",
"\n",
"Epoch 00033: loss did not improve from 2.89863\n",
"Epoch 34/50\n",
"1968/1968 [==============================] - 13s 7ms/step - loss: 2.9013\n",
"\n",
"Epoch 00034: loss did not improve from 2.89863\n",
"Epoch 35/50\n",
"1968/1968 [==============================] - 14s 7ms/step - loss: 2.9019\n",
"\n",
"Epoch 00035: loss did not improve from 2.89863\n",
"Epoch 36/50\n",
"1968/1968 [==============================] - 13s 7ms/step - loss: 2.9015\n",
"\n",
"Epoch 00036: loss did not improve from 2.89863\n",
"Epoch 37/50\n",
"1968/1968 [==============================] - 13s 6ms/step - loss: 2.9002\n",
"\n",
"Epoch 00037: loss did not improve from 2.89863\n",
"Epoch 38/50\n",
"1968/1968 [==============================] - 13s 7ms/step - loss: 2.9011\n",
"\n",
"Epoch 00038: loss did not improve from 2.89863\n",
"Epoch 39/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9023\n",
"\n",
"Epoch 00039: loss did not improve from 2.89863\n",
"Epoch 40/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.8997\n",
"\n",
"Epoch 00040: loss did not improve from 2.89863\n",
"Epoch 41/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.8998\n",
"\n",
"Epoch 00041: loss did not improve from 2.89863\n",
"Epoch 42/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9002\n",
"\n",
"Epoch 00042: loss did not improve from 2.89863\n",
"Epoch 43/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9005\n",
"\n",
"Epoch 00043: loss did not improve from 2.89863\n",
"Epoch 44/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9021\n",
"\n",
"Epoch 00044: loss did not improve from 2.89863\n",
"Epoch 45/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9007\n",
"\n",
"Epoch 00045: loss did not improve from 2.89863\n",
"Epoch 46/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9007\n",
"\n",
"Epoch 00046: loss did not improve from 2.89863\n",
"Epoch 47/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9018\n",
"\n",
"Epoch 00047: loss did not improve from 2.89863\n",
"Epoch 48/50\n",
"1968/1968 [==============================] - 12s 6ms/step - loss: 2.9012\n",
"\n",
"Epoch 00048: loss did not improve from 2.89863\n",
"Epoch 49/50\n",
"1968/1968 [==============================] - 13s 7ms/step - loss: 2.9017\n",
"\n",
"Epoch 00049: loss did not improve from 2.89863\n",
"Epoch 50/50\n",
"1968/1968 [==============================] - 13s 6ms/step - loss: 2.9017\n",
"\n",
"Epoch 00050: loss did not improve from 2.89863\n"
]
},
{
"data": {
"text/plain": [
"<keras.callbacks.History at 0x7f0868b2a3c8>"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"filename = \"data.txt\"\n",
"raw_text = open(filename).read()\n",
"raw_text = raw_text.lower()\n",
"# create mapping of unique chars to integers\n",
"chars = sorted(list(set(raw_text)))\n",
"char_to_int = dict((c, i) for i, c in enumerate(chars))\n",
"# summarize the loaded data\n",
"n_chars = len(raw_text)\n",
"n_vocab = len(chars)\n",
"print(\"Total Characters: \", n_chars)\n",
"print(\"Total Vocab: \", n_vocab)\n",
"# prepare the dataset of input to output pairs encoded as integers\n",
"seq_length = 100\n",
"dataX = []\n",
"dataY = []\n",
"for i in range(0, n_chars - seq_length, 1):\n",
" seq_in = raw_text[i:i + seq_length]\n",
" seq_out = raw_text[i + seq_length]\n",
" dataX.append(encodeEsp(seq_in))\n",
" dataY.append(encodeEsp(seq_out))\n",
"n_patterns = len(dataX)\n",
"print(\"Total Patterns: \", n_patterns)\n",
"\n",
"# one hot encode the output variable\n",
"y = np.array(dataY)\n",
"x = np.array(dataX)\n",
"y = y[:,0,:]\n",
"print(x.shape)\n",
"print(y.shape)\n",
"# define the LSTM model\n",
"def Model(seq_len):\n",
" model = Sequential()\n",
" model.add(GRU(256, input_shape=(seq_len, 30), return_sequences=True))\n",
" model.add(Dropout(0.1))\n",
" model.add(GRU(256))\n",
" model.add(Dropout(0.1))\n",
" model.add(Dense(y.shape[1], activation='softmax'))\n",
" return model\n",
"model = Model(100)\n",
"model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
"model.summary()\n",
"# define the checkpoint\n",
"filepath=\"weights-improvement-{epoch:02d}-{loss:.4f}.hdf5\"\n",
"checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')\n",
"callbacks_list = [checkpoint]\n",
"# fit the model\n",
"model.fit(x, y, epochs=50, batch_size=64, callbacks=callbacks_list, shuffle=True)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(1, 22, 30)\n",
" "
]
}
],
"source": [
"import sys\n",
"import numpy as np\n",
"int_to_char = dict((i, c) for i, c in enumerate(chars))\n",
"start = numpy.random.randint(0, len(dataX)-1)\n",
"pattern = encodeEsp(\"Mi mangas multajn pomo\")\n",
"pattern = np.array([pattern])\n",
"print(pattern.shape)\n",
"m2 = Model(pattern.shape[1]) # To word arround Keras's static graph limitation\n",
"m2.set_weights(model.get_weights()) \n",
"m2.compile(loss='categorical_crossentropy', optimizer='adam')\n",
"for i in range(1000):\n",
" out = m2.predict(pattern)\n",
" print('{}'.format(decodeEsperanto(out.argmax())), end='')\n",
" pattern = pattern[:, 1:]\n",
" out = np.array([out])\n",
" pattern = np.concatenate((pattern, out), axis=1)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment