Kaggle-mnist-CNN-ES.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Kaggel: Digit Recognizer(mnist) , score:0.99528"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import tensorflow as tf\n",
    "import random as rn\n",
    "import os\n",
    "os.environ['PYTHONHASHSEED'] = '0'\n",
    "seed = 123\n",
    "rn.seed(seed)\n",
    "np.random.seed(seed)\n",
    "session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)\n",
    "from keras import backend as K\n",
    "tf.set_random_seed(seed)\n",
    "sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)\n",
    "K.set_session(sess)\n",
    "\n",
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42000, 785)"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28000, 784)"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = train.label\n",
    "label.shape\n",
    "test_index = test.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42000, 784)"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = train.drop(['label'], axis=1)\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.values.reshape(-1,28,28,1) / 255.0\n",
    "test = test.values.reshape(-1,28,28,1) / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABIEAAACBCAYAAABXearSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHfFJREFUeJzt3XmUVMXZx/EaB4GwCAmMbLIYQ4JhjbJ5WISAAgFZgqC4oCJE2YyoLB4OiIGwRDASQRBQWQ0RA8gmqxgYEAGVCLLoGBEUAdn3ffKH5zw+VXY3d2b69nT3/X7++tVbt29XaKv7zn3vU5WSmZlpAAAAAAAAkNyuye0BAAAAAAAAwH/cBAIAAAAAAAgAbgIBAAAAAAAEADeBAAAAAAAAAoCbQAAAAAAAAAHATSAAAAAAAIAA4CYQAAAAAABAAHATCAAAAAAAIAC4CQQAAAAAABAAeWL5ZikpKZmxfD/8KDMzMyUa5+EzzD3R+gyN4XPMTczFxMdcTA7MxcTHXEwOzMXEx1xMDszFxOf1M+RJIAAAAAAAgADgJhAAAAAAAEAAcBMIAAAAAAAgALgJBAAAAAAAEADcBAIAAAAAAAgAbgIBAAAAAAAEADeBAAAAAAAAAoCbQAAAAAAAAAHATSAAAAAAAIAA4CYQAAAAAABAAHATCAAAAAAAIAC4CQQAAAAAABAA3AQCAAAAAAAIgDy5PYAgW7lypeQmTZpYfQ899JDk6dOnx2xMieIXv/iF5EKFCll9PXv2DPmaOnXqWO1XXnlF8okTJ6y+ZcuWSc7MzMz2OJF9qampkv/2t79JvnLlinXcgAEDJF++fNn/gQFJIiUlRXLJkiUl9+jRwzquVKlSkh999FFP537jjTes9pAhQyR/8803kt35DH+E+z5t0KCBdVzNmjUlr1271urTv63btm2L9hABAPDk2muvlez+fdeqVauQrylYsKDV1r9p+npow4YN1nGzZ8+WPGPGDKvv7NmzIXMi4EkgAAAAAACAAOAmEAAAAAAAQACkxLLUJSUlJdB1NatXr7ba9erVk6wf1TbGmIcffliy++hZdmRmZqZc/airi+VnWLhwYavdokULyTNnzpScJ0/2qhq/+OILyWXLlrX6pk2bJnnUqFGSd+/ena33ioZofYbGJMZc/NnPfib59OnTYY8rUKCA5HPnzvk6pmiI17mYkZFhtXfs2CG5ffv2Vt+FCxei+dYR6f8OmjZtKnnhwoUxG4MrUedi/vz5rbYuO54wYUKshmGefvppyWPHjrX6YlkeFq9zMRr0o/LGGDN16lTJnTp1krx48WLruGPHjknu2LGj1afnfYcOHSQvXbo0R2PNiUSdi7Al81wMiiDPxUqVKknu3bu31ZcvXz7JJUqUsPpatmwZ8nybNm2y2nPnzpX87rvvWn2ffvpp1gZ7FfE6F3VZujHGPPfcc5K7desWzbfKEj2OYcOG5do4NK+fIU8CAQAAAAAABAA3gQAAAAAAAAKAcjCfDRw4UPKgQYOsPv249ltvvWX16R1Yzpw5k+NxxOvjfa6iRYtKdsvgwj026bcDBw5IbtOmjdW3a9cuycePH/d1HEF71JZysMii/RnecMMNVluXS5YuXdrqO3r0aDTfOqIyZcpInjdvnuTatWvHbAyuRJqLejeM9evXW31Vq1b18609cR+dHz9+fMzeO17nYjSMGDHCavfv31/yxIkTJbs7wWmrVq2y2o0bN5asv5OrVKliHff1119nbbA5kEhzEeElw1xMS0uTrL/X6tevbx3XqFGjkK+/dOmS1dalmjt37rT69LWnNn/+fKt96tSpsOePtmSfi+4SFcOHD5fcuXNnye6OxZregcqY7O0+7F7nzpkzR7JeSiS74nUuur9p9913n2S9Y7Qx9t8FH330kWS33Pzw4cOSjxw5IrlWrVrWcRUrVgw7ru3bt0tOT0+X3L1797Cv8RvlYAAAAAAAABDcBAIAAAAAAAgAbgIBAAAAAAAEAGsC+aBt27aS//nPf0rOmzevddzWrVslN2jQwOo7efJkVMcUrzWerubNm0tesmSJn28VFXo9Bb3Ogh+Svd7a5XVNoJ49e0qO5TbX2ZUoc/HEiROS//Wvf1l9sdyOU68JtHfvXsl6fRJjjPnPf/4TszEl0lwsX7685K+++srPt8qWzz//3GqPGTNG8uuvv271Xb58OarvnShz0at27dpJ1tcexthriNSsWVPyxYsXw57PXZevRYsWkvUaDH379rWO05+h3+J5LurPwxhjmjVrJlmvb2aMMYcOHQp5jj179ljtYsWKSdbrfWVFw4YNJevr1R07dljH6TVP3HFEW7zORXc9vFatWkm+++67rb6mTZuGPMeFCxes9r59+0Iel5qaarXLli3reZzhbNmyRfL06dOtvnHjxkmOxnpB8TwXs0v/frrXGOE+H/dvF/0dG401gX73u99Z7ZIlS0qeNGmSZPd72f3vMJx4nYuucuXKSdZr3hljzLJlyyTrtbW8XkMUL17caj/11FNh30vT16gVKlTw9F5+YE0gAAAAAAAACG4CAQAAAAAABADlYFHgPhK4YMECydWqVZOst58zxpguXbpIXrhwoU+j+0G8Pt7nbp05dOhQybfffnuOz//nP/9ZsvsI7jPPPCO5Tp062Tq/LlPSn6cx9raN0ZCMj9pG4rUcbPny5ZJ1OWG8ite56Jo6dark6tWrW316vnh9xDi7wpWDNWnSxDpu9erVvo5Di+e5WKJECau9cuVKyZUrV/Z0DrdESJcDuqXLmn4sPV++fJ7eK5Kbb77ZaofbFjm7EmUuRpI/f37JmzZtkux+1vq3dv369dl6L/14uz6H3mbXGGNuvfVWyX5/P8TzXHz22Wet9rBhwyS71966TET36e88Y+wyBb0NstfzRepzP0e9TXJQy8E++eQTq+3+Fmr6Ol5vFa3/JjAm/PdY3bp1rfb7778v+YknnrD6Nm7cGPIc7rVsp06dJOsyQGOMGTVqlGT3v9XsiOe5mBX6t+u9996TfNttt1nH6bkze/ZsyQ8++KB1nLsteU65W9DrrdL/+Mc/Sr733nut444dO+bp/PE6F2PphhtusNp6Dkf6DqAcDAAAAAAAAHGHm0AAAAAAAAABwE0gAAAAAACAAMiT2wNIVLVr15Y8efJkq69KlSohX9O7d2+r7fc6QIngySeftNpe1wHavHmz5A8//DDscXqdkG3btll9S5culay3uzXGXs9Hf9YuvUVrx44dw54DSDR6O/HOnTtbfUWKFJH8/fff+zqO8+fPSz5+/Liv75UM9FamxnhfB2j//v2SH3vsMavP62/VnXfeKXn8+PFW30033eTpHNo777xjtfWacbNmzcry+ZKRXvdOf9avv/66dVyk30mvTpw4EfL/7v43prfV3r17d47fN1Fdc439/2ft3r275DVr1lh97not0eSuvfjAAw+EPM6dU36vA5QIRo8ebbX1mkx662ljjMnIyMjRexUtWtRqd+3aVfLMmTM9nUNvCW+M/Zm618AtW7aUPHjwYMnumnBBM3bsWMnuOk2a/rfVf8tEew0g16lTp6y23hZeZ2TfLbfcYrUjrQOUyHgSCAAAAAAAIAC4CQQAAAAAABAAlIN55G75N23aNMnu9pu6ZEFvz7ts2TKfRpdY9Pak7uPSkdx///2SDx48KHnVqlXZGofedtzdglyXitWsWVNypPFWqlTJardq1UryokWLsjVGILd8/PHHuT0EY4wxhw4dkuw+zo4fXHvttZJbt26drXN8+eWXkrNbqrx8+XLJY8aMsfr0FsRly5b1dL5f//rXVnvQoEGSdTmNu412MtPbghsTvrRnxIgRVvvy5cs5fu/rrrtOcsmSJXN8vmTXtm1bq62XDti5c6fV57ajqV27dlZbX7Nu375d8vDhw30bQ6KKZdmpvu7MCl26oreEN8aYbt26Sdbz1xhjmjRpIjnoJWBa+/btJeu/V6ZOnWod16dPH8mUqic+fR2VlpZm9R0+fFhysWLFYjYmv/EkEAAAAAAAQABwEwgAAAAAACAAKAeLoESJEpL79u3r+XV6R5NHHnkkqmNKBtWqVZPsPi4dSXp6umS/H/8fMmSI5K1bt0qOtOOXu0PKXXfdJZlysKzT5QsrVqyQfMcdd+TGcAJH78oVj/T8MsbeCTBo9A5Rv/nNbzy/7sKFC5JHjhwZ1TFNnDjRai9YsEDyvHnzJNeqVcvzOXV5mC61dr97L1265PmciaZHjx5WW/9vnzJliuQg78oVr9yScT/pnUvLlStn9ekSFz3vdektcl++fPkku7s+Pvroo5J/+ctfSnaXNvjkk08ku7+ZlDD9oEWLFlZb736qSyd1+Zcx3v/99K5vefLYf3br8+uSI/jDLYnUf+vpUnp3mRevS5foz1qXwBtjzIsvvig5Xq6veRIIAAAAAAAgALgJBAAAAAAAEADcBAIAAAAAAAgA1gRy6Ho+vd2tu+aAdvLkSaut1z7AT914442ejjtx4oTVzq0tLNevXy/ZHZNbX4ro0euV6K05WRMoNvR/69HYXjraOnToYLXdNROC5IUXXpDs1rJHsmnTJsmLFy+O6phc+/btk6y3rNbrAxnjfY2gihUrStZrnCS7/Pnzh+3btWuXZD/mrF4/QXPXxjh79mzU3ztR6HV/3DWA9BbxsRyHu07Y3LlzJbvzD5Hp+afX5THG3mI6ku+++05yqVKlJJctW9Y6Tq/h467rtGzZMsmPP/645C1btljHsc5TaHq9pcGDB1t9qampIV8TaQ0g/Tl2797d6tNtd3txvTbMpEmTJLvr0OrrYWRfgQIFrLZeT9GrI0eOWO0rV65ILl68uORhw4ZZxzVq1EjywIEDrb7NmzdneRzRwJNAAAAAAAAAAcBNIAAAAAAAgACgHMyht9WsUqWKp9e4j3C65WGwHTt2zNNxGzdutNpHjx71YzhXpR/dXbJkidV37733hn1ds2bNJBcqVEjyqVOnoji65KW30rzttttycSTBtGHDBsl79+61+vRjrr169ZLsd8mmLlkaMGCA1Ve4cGHJfAd7o8ssY0mXhrVt29bq01saX3/99Z7OV758eaudkZGRg9HFtzZt2oTtmz9/vq/vrUvwtLVr11rtAwcO+DqORJGbpTgzZsyQ7JZL6qUOzpw5E7MxJQNdju6WIHtd6iAc93d2xIgRklevXm316dJPZJ2+Xqhdu3bY4xYtWiS5a9euVl///v0lp6WlhTz31eTNm1eyvpZyvzuGDh3q+ZwIz/3789VXX5Xs9W9+d97rv+n03ypTpkyxjmvatKlkt7SwY8eOnt472ngSCAAAAAAAIAC4CQQAAAAAABAAgS8H0yt5G2PMwoULJUfacUSXSrBq+9XpXbRmz57t6TX60Tlj7NIA97HZWJk1a5bVjlQOpndz8LprBH6k/830Y7KIvW7dulntpUuXSv773/8ueefOnb6OQ5cRFSlSxOqrW7eu5BUrVvg6DkSPLrc1xphz585l+RydO3e22u5uL4muRIkSkn/1q19ZfV999ZXk/fv3+zoOfU2k84cffujr+yYS/R3odac7P+gdwbKyYyAi038jrFq1yurzWr6qdenSRbK74+X9998v+YMPPsjyuRGeLsdJT0+3+urXry+5VatWkvVubcaEn1d6101jjNm6dWvYcbRv316yvqbRO74ZY+8cRrlt9rnXFz169Ijq+fXOp4mAJ4EAAAAAAAACgJtAAAAAAAAAAcBNIAAAAAAAgAAI/JpA48aNs9rVq1eXrOs9169fbx2n16s5f/68T6NLHnq77+zUTceLb7/9NreHAMScu/bB0aNHJb/00kuSmzdv7us49BbxbG2cnPS29cm2tk80uOtQfPbZZ5JPnz4d1fcqUKCA1dbbIOtx8LsYWiy3iG/YsKHVjrSm5Zo1a/weTiC4v0G7d+/O8jn0d9ywYcOsvu7du0vW6/AZY68RdM8990i+ePFilscQRPrfaeDAgVafvt7Ra1OePHnSOk6vETpy5EjJe/bs8TwOvf5Q0aJFJZcqVco67qabbpLMmkDxpU6dOpKfeeaZXBxJ1vEkEAAAAAAAQABwEwgAAAAAACAAAlkOpreF14/YufTjgqNGjbL6KAHLmmPHjknWj1DqLTABJB691arf9PfIp59+avX16dNH8rp166w+SscSR6FChbL8mh07dvgwkviRL18+yQULFrT6Spcu7dv76i2LjbHLFbT//e9/vo0B3lSqVMlq63K9uXPnWn16G3v8lF4WYu/evVbfkSNHfHvfCxcuWO2xY8dKXrZsmdW3YsUKyRs2bJDcsWNH67gvv/wymkNMSu4W8ZUrV5acmpoq+ezZs9ZxWSn7CkfPU50PHz5sHUfJbfxq2bKl5HC/kfGKJ4EAAAAAAAACgJtAAAAAAAAAAcBNIAAAAAAAgAAIzJpAelvyN998U/Itt9xiHXfu3DnJjz/+uORFixb5OLrkd+XKFcm6ljkrawLNmTNHctOmTSWfOnUqh6OLTNd4Tps2zfPrJk6cKFmvZQIkuvnz50u+9dZbJefJY/+kXLp0KeTr3XVMqlWrJrlu3bpWn6631tu16te4nn32Was9aNCgsMcid7Vu3dpq9+7dO8vnePvtt6M1nLik55G7boiffv/731vtYsWKhRzHvn37YjYmhNagQQOrrbeI19/XCE3/jaCvURs1amQd5+eaQJG46zh16NBB8uTJkyWvXr3aOk5fK3/++ec+jS65ZGRk+HZud+2uMmXKhDzuo48+stpff/21b2PCT7nXsvnz55fcq1cvq+/222/3dM5du3ZJfuqpp3IwuujhSSAAAAAAAIAA4CYQAAAAAABAAASmHKxdu3aSGzduHPa4jRs3Sp4xY4avYwqqd955R/KWLVusvho1aoR9Xe3atSW/9957kvv3728d5z4Omx1paWmSR48eLblq1aphX+NuHzlq1CjJeutHINFNnz5dcteuXSW7ZVe6DLJFixaS69WrZx2XN29eyWvWrLH6hgwZIllvm9q2bVvruH79+klev359xPHjB/rfTH9v+r3ld4UKFSTrcj9j7JK/SHTZWLiyw2Sh54e7RXy0NWnSRPIrr7wS9rgxY8ZI9rN8At5E2iJ+x44dsR5OwvnDH/4geeHChZK3b9+eG8O5Kr0tvP4OdbeS13P4rrvusvrca1b4z11SolChQiGPmzdvXiyGAyVfvnyS//GPf1h9+jrXK/e7Q8/Tb775Jsvn8wNPAgEAAAAAAAQAN4EAAAAAAAACIGnLwTp16mS1dWmO5pYN3Hfffb6NCT84fvy45CeeeMLqmzBhguTKlSuHPUfNmjUlP//881bf0aNHQ77mxIkTVls/Yq9XfjfGfmQzUgmYtmTJEqvNav458/LLL+f2EBDG1q1bJesdR/SOii49P55++mmrb/PmzSFzJO4uLbq0KWh0WW316tU9v65ixYqSe/bsKdn9fLKjXLlyVlt/1z/00EOS9Y5TV/Paa69J1r8VQS63LVCggGT9OPv58+c9vd7dIVWXIbilCunp6ZL5fs59emdG93PUu4Mha/Q1aiLYs2eP5Oeee87qmz17tmS3DHvlypX+DgzGGGP69OkjuVatWlaf/u3Sv29vvPGG/wNLMLqM3Bj7elMvEWKMMe+//75kvZOluxubLqPVS4vosuis0EsWtGrVyurT8zRe8CQQAAAAAABAAHATCAAAAAAAIAC4CQQAAAAAABAASbUmUJEiRSQPHTrU6itcuHDI1+htTo0x5rvvvov+wBCWXmPAGPtz0/WxxoTfGrd+/fpW++OPPw553Pfff2+19VoK0dh2d86cOTk+B35UtmxZyaxvEF/0mgnu1sSxcujQoVx533jUuHFjyW5tfI0aNTydQ6/Z07RpU6tv4sSJns7x8MMPS9brDRljTNGiRT2dQ9u2bZvVHjhwoOQrV65k+XyJ6ttvv5W8du1aq69BgwaSmzVrJnnBggVhz6fXYWrdurXVp9cBWrdundXXpUsXyfv377/asBFDQV4XKxr0tX+PHj0k678rjIn/9YLmz59vtXfu3Cm5ffv2Vh9rAvmjYcOGVlv/neley548eVLysGHDJF+8eNGn0SWW0qVLS/7ggw+svuuvv15y3759rb41a9ZIPnfunGT3etVdu9ALvW6eMcZUqVJFsr5WSoR1YXkSCAAAAAAAIAC4CQQAAAAAABAASVUO1qZNG8k33nijp9dcd911fg0H2fDWW29Jdrfyc0v3siotLS1Hrzfmp48CP/bYY5IXL16c4/MjNB51B8I7duyYZLcU+t///renc6SmpkquWrWq1Td+/PgcjC5rdAmYW5Z28ODBmI0jnujSgDfffNPq0+VgL730UsjXGGPMnXfeKfmBBx6QrEvDjLFLz/T5jDEmIyMjK8NGDLllJpRQZ40us9Sl6LrE0hhj3n77bcnxWJKqt8M2xpgDBw5Irlu3bqyHk9T0khK9evWS7JYm6etX93u5X79+kuNxC/Hcpv+N9XWOMXY5mMstyYsm9xpLX5ck2pIyPAkEAAAAAAAQANwEAgAAAAAACABuAgEAAAAAAARAUq0JpGst3Vrda6758X7X5cuXJbvb2CJ+TJkyxWrfcccdkps3bx6zcZw+fVryPffcY/UtX748ZuMA8CO9taoxxmzZskVyhQoVYjya+OFuEfzggw9KnjFjRqyHE5Lettitr587d67k8+fPx2xMieLdd9+12qdOnZKs/7v3ukade63Up08fyV7Xk0Luc9fN03NMZ4R25swZyXqdlunTp1vHVa5cWfLw4cOtvnj4vnLXo6levbrkv/zlL7EeTkKoU6eO1dbbkuvtwP/0pz9Zx/Xu3Vvyb3/7W0/v9eKLL1rtV1991fM4g0hvsz5kyBCrb+TIkZK9bvWu57kxxsyaNUuyu/6Xpj+3zz77zOq7dOmSp/eORzwJBAAAAAAAEADcBAIAAAAAAAiAlFhuvZySkhKzN9u+fbvVzpPnx8q3v/71r5KnTZsWqyHlqszMzKjsFxrLz9CVP39+yXrrYL31rTH2Vo16m1T3v3Xd9/LLL1t9zz//vGT9qJ+7RXwsReszNCZ3P0evbr/9dsmrV68Oe1yjRo0kr1mzxs8hRUUyzMV4pEsz9TbXxhjzyCOPRPW9Emku6u+5n//851bfk08+KblNmzaS3S3ivdKlE+52tzt27JA8Z84cybn5KHUyzMUSJUpIvvnmmyV37tzZOk6XK+zbt0+yW56Qnp4e7SH6KpHmYrRNnDhRcrdu3ay+u+++W7IuaYlX8ToX3Xk0adIkybt27bL6BgwYIFlvOa9LNqNFz+fu3buHzMYYM3r0aMn6utYYY86ePRvVMSXqXNS/fcYYM3PmTMn636h48eLWceH+fv7iiy+s9muvvSb5hRdeyPY4YyVe52KTJk2s9tGjRyUPHjzY6tu7d6/kFStWSHb/RtDbzhcqVCjse/sxh/3k9TPkSSAAAAAAAIAA4CYQAAAAAABAACRtORhs8fp4H7xL1EdtYWMuRkfevHmt9qZNmySPGzfO6ps8eXJU35u5mByYi4kvyHPxwIEDkosVK2b16SUQEkGizMUaNWpI1uW0xti7TBUpUkTy0qVLreN0OawuN3J3OKpXr55kd9mDMmXKSM7IyJDsLm0wYcKEEP8r/JGoc1F/psYYs27dOsl6GQpdWm2MMf/9738l65JLXf5lzE/L0+NdosxFhEc5GAAAAAAAAAQ3gQAAAAAAAAKAm0AAAAAAAAABwJpAAUGNZ+JL1Hpr2JiLiY+5mByYi4kvaHMxLS1N8sGDByVfuXLFOi41NTVmY4qGZJiLBQsWlNyvXz/J9evXt46rWrWq5DNnzkguX768dZzeZj49Pd3q0+vW6C2wL1y4kNVhR03Q5mKySoa5GHSsCQQAAAAAAADBTSAAAAAAAIAAoBwsIHi8L/HxqG1yYC4mPuZicmAuJr6gzcXixYtL1lvEb9++3TpOlxwlAuZi4gvaXExWzMXERzkYAAAAAAAABDeBAAAAAAAAAoCbQAAAAAAAAAGQJ7cHAAAAACCyQ4cOSU60beABAPGDJ4EAAAAAAAACgJtAAAAAAAAAARDTLeIBAAAAAACQO3gSCAAAAAAAIAC4CQQAAAAAABAA3AQCAAAAAAAIAG4CAQAAAAAABAA3gQAAAAAAAAKAm0AAAAAAAAABwE0gAAAAAACAAOAmEAAAAAAAQABwEwgAAAAAACAAuAkEAAAAAAAQANwEAgAAAAAACABuAgEAAAAAAAQAN4EAAAAAAAACgJtAAAAAAAAAAcBNIAAAAAAAgADgJhAAAAAAAEAAcBMIAAAAAAAgALgJBAAAAAAAEADcBAIAAAAAAAgAbgIBAAAAAAAEADeBAAAAAAAAAoCbQAAAAAAAAAHATSAAAAAAAIAA+D/1sc807iDEqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8a75d310f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(20,2))\n",
    "n = 10\n",
    "for i in range(n):\n",
    "    plt.subplot(1,n,i+1)\n",
    "    plt.imshow(train[i,:,:,0], cmap='gray')\n",
    "    plt.axis('off')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential, Input, Model\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "import keras\n",
    "initializer = keras.initializers.glorot_uniform(seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, kernel_size=(5,5),padding='Same', activation='relu', input_shape=(28,28,1), kernel_initializer=initializer))\n",
    "model.add(Conv2D(32, kernel_size=(5,5),padding='Same', activation='relu',kernel_initializer=initializer))\n",
    "model.add(MaxPool2D(pool_size=(2,2)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.25, seed=seed))\n",
    "model.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', activation ='relu', kernel_initializer=initializer))\n",
    "model.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', activation ='relu', kernel_initializer=initializer))\n",
    "model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.25, seed=seed))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128, activation = \"relu\", kernel_initializer=initializer))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.5, seed=seed))\n",
    "model.add(Dense(64, activation = \"relu\", kernel_initializer=initializer))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.5, seed=seed))\n",
    "model.add(Dense(10, activation = \"softmax\", kernel_initializer=initializer))\n",
    "\n",
    "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/100\n",
      "33600/33600 [==============================] - 14s 405us/step - loss: 0.4613 - acc: 0.8589 - val_loss: 0.0701 - val_acc: 0.9779\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.07012, saving model to cnn_final_checkpoint.hdf5\n",
      "Epoch 2/100\n",
      "33600/33600 [==============================] - 11s 337us/step - loss: 0.1473 - acc: 0.9584 - val_loss: 0.0528 - val_acc: 0.9836\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.07012 to 0.05284, saving model to cnn_final_checkpoint.hdf5\n",
      "Epoch 3/100\n",
      "33600/33600 [==============================] - 11s 338us/step - loss: 0.1070 - acc: 0.9683 - val_loss: 0.0420 - val_acc: 0.9864\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.05284 to 0.04198, saving model to cnn_final_checkpoint.hdf5\n",
      "Epoch 4/100\n",
      "33600/33600 [==============================] - 11s 339us/step - loss: 0.0920 - acc: 0.9738 - val_loss: 0.0491 - val_acc: 0.9849\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.04198\n",
      "Epoch 5/100\n",
      "33600/33600 [==============================] - 11s 341us/step - loss: 0.0834 - acc: 0.9760 - val_loss: 0.0522 - val_acc: 0.9850\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.04198\n",
      "Epoch 6/100\n",
      "33600/33600 [==============================] - 11s 340us/step - loss: 0.0744 - acc: 0.9793 - val_loss: 0.0396 - val_acc: 0.9898\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.04198 to 0.03960, saving model to cnn_final_checkpoint.hdf5\n",
      "Epoch 7/100\n",
      "33600/33600 [==============================] - 11s 340us/step - loss: 0.0689 - acc: 0.9815 - val_loss: 0.0306 - val_acc: 0.9915\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.03960 to 0.03064, saving model to cnn_final_checkpoint.hdf5\n",
      "Epoch 8/100\n",
      "33600/33600 [==============================] - 11s 342us/step - loss: 0.0601 - acc: 0.9835 - val_loss: 0.0332 - val_acc: 0.9912\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.03064\n",
      "Epoch 9/100\n",
      "33600/33600 [==============================] - 11s 341us/step - loss: 0.0566 - acc: 0.9848 - val_loss: 0.0558 - val_acc: 0.9842\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.03064\n",
      "Epoch 10/100\n",
      "33600/33600 [==============================] - 11s 341us/step - loss: 0.0542 - acc: 0.9851 - val_loss: 0.0411 - val_acc: 0.9894\n",
      "\n",
      "Epoch 00010: ReduceLROnPlateau reducing learning rate to 0.00020000000949949026.\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.03064\n",
      "Epoch 11/100\n",
      "33600/33600 [==============================] - 11s 339us/step - loss: 0.0384 - acc: 0.9902 - val_loss: 0.0242 - val_acc: 0.9939\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.03064 to 0.02415, saving model to cnn_final_checkpoint.hdf5\n",
      "Epoch 12/100\n",
      "33600/33600 [==============================] - 11s 340us/step - loss: 0.0294 - acc: 0.9921 - val_loss: 0.0242 - val_acc: 0.9936\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.02415\n",
      "Epoch 13/100\n",
      "33600/33600 [==============================] - 12s 352us/step - loss: 0.0279 - acc: 0.9921 - val_loss: 0.0246 - val_acc: 0.9938\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.02415\n",
      "Epoch 14/100\n",
      "33600/33600 [==============================] - 12s 346us/step - loss: 0.0257 - acc: 0.9926 - val_loss: 0.0249 - val_acc: 0.9932\n",
      "\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 4.0000001899898055e-05.\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.02415\n",
      "Epoch 15/100\n",
      "33600/33600 [==============================] - 12s 346us/step - loss: 0.0233 - acc: 0.9936 - val_loss: 0.0224 - val_acc: 0.9942\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.02415 to 0.02237, saving model to cnn_final_checkpoint.hdf5\n",
      "Epoch 16/100\n",
      "33600/33600 [==============================] - 12s 345us/step - loss: 0.0229 - acc: 0.9934 - val_loss: 0.0225 - val_acc: 0.9944\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.02237\n",
      "Epoch 17/100\n",
      "33600/33600 [==============================] - 12s 346us/step - loss: 0.0206 - acc: 0.9950 - val_loss: 0.0228 - val_acc: 0.9940\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.02237\n",
      "Epoch 18/100\n",
      "33600/33600 [==============================] - 12s 345us/step - loss: 0.0209 - acc: 0.9940 - val_loss: 0.0225 - val_acc: 0.9944\n",
      "\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 8.000000525498762e-06.\n",
      "\n",
      "Epoch 00018: val_loss did not improve from 0.02237\n",
      "Epoch 19/100\n",
      "33600/33600 [==============================] - 12s 346us/step - loss: 0.0200 - acc: 0.9944 - val_loss: 0.0224 - val_acc: 0.9944\n",
      "\n",
      "Epoch 00019: val_loss did not improve from 0.02237\n",
      "Epoch 20/100\n",
      "33600/33600 [==============================] - 12s 344us/step - loss: 0.0196 - acc: 0.9946 - val_loss: 0.0225 - val_acc: 0.9943\n",
      "\n",
      "Epoch 00020: val_loss did not improve from 0.02237\n",
      "Epoch 21/100\n",
      "33600/33600 [==============================] - 12s 345us/step - loss: 0.0208 - acc: 0.9944 - val_loss: 0.0226 - val_acc: 0.9944\n",
      "\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.6000001778593287e-06.\n",
      "\n",
      "Epoch 00021: val_loss did not improve from 0.02237\n",
      "Epoch 22/100\n",
      "33600/33600 [==============================] - 12s 345us/step - loss: 0.0204 - acc: 0.9947 - val_loss: 0.0226 - val_acc: 0.9943\n",
      "\n",
      "Epoch 00022: val_loss did not improve from 0.02237\n",
      "Epoch 23/100\n",
      "33600/33600 [==============================] - 12s 345us/step - loss: 0.0192 - acc: 0.9946 - val_loss: 0.0225 - val_acc: 0.9944\n",
      "\n",
      "Epoch 00023: val_loss did not improve from 0.02237\n",
      "Epoch 24/100\n",
      "33600/33600 [==============================] - 12s 347us/step - loss: 0.0191 - acc: 0.9954 - val_loss: 0.0227 - val_acc: 0.9942\n",
      "\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 1e-06.\n",
      "\n",
      "Epoch 00024: val_loss did not improve from 0.02237\n",
      "Epoch 25/100\n",
      "33600/33600 [==============================] - 12s 343us/step - loss: 0.0200 - acc: 0.9943 - val_loss: 0.0225 - val_acc: 0.9944\n",
      "\n",
      "Epoch 00025: val_loss did not improve from 0.02237\n",
      "Epoch 00025: early stopping\n"
     ]
    }
   ],
   "source": [
    "epochs = 100\n",
    "reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=3, min_lr=0.000001,verbose=1)\n",
    "early_stopping = EarlyStopping(patience=10, verbose=1)\n",
    "checkpointer = ModelCheckpoint(filepath='cnn_final_checkpoint.hdf5', verbose=1, save_best_only=True)\n",
    "\n",
    "hist = model.fit(train, label, \n",
    "                          batch_size=32, \n",
    "                          epochs=epochs,\n",
    "                          verbose=1,\n",
    "                          validation_split=0.2,\n",
    "                          callbacks=[reduce_lr, early_stopping, checkpointer])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Epoch 00024: ReduceLROnPlateau reducing learning rate to 1e-06.\n",
    "\n",
    "# Epoch 00024: val_loss did not improve from 0.02237\n",
    "# Epoch 25/100\n",
    "# 33600/33600 [==============================] - 12s 343us/step - loss: 0.0200 - acc: 0.9943 - val_loss: 0.0225 - val_acc: 0.9944\n",
    "\n",
    "# Epoch 00025: val_loss did not improve from 0.02237\n",
    "# Epoch 00025: early stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD8CAYAAACYebj1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd4FNX6wPHvm5ACJCAJoUhHQOmgEUVUUK8KFkBApVjQa7s/sVfEgihiwQpcuVzBK4oiogjeq1gQBBSVIJ2IYKQEAgRCSQgh7f39MZuwhJBskk02m30/zzNPdmfOnD0zLPPunHPmHFFVjDHGGIAgXxfAGGNM5WFBwRhjTD4LCsYYY/JZUDDGGJPPgoIxxph8FhSMMcbks6BgjDEmnwUFY4wx+SwoGGOMyVfN1wUoibp162rz5s19XQxjjPErK1as2KuqMZ6k9aug0Lx5c+Li4nxdDGOM8SsistXTtFZ9ZIwxJp8FBWOMMfksKBhjjMlnQcEYY0w+CwrGGGPyWVAwxhiTz4KCMcaYfB4FBRHpLSIbRWSziDxeyPZmIrJARNaIyCIRaey27SURWedarndb/x8R+UtEVrmWLt45JGOMqXwOHYJZs2DqVEhL83VpTq7YoCAiwcAkoA/QDhgiIu0KJBsPTFfVTsAYYJxr3yuBM4EuwDnAIyJSy22/R1S1i2tZVeajMcaYSmTXLpgyBa64AmJi4Prr4bbboGlTGDXK2V7ZeHKn0A3YrKoJqpoJzAT6FUjTDljger3QbXs74AdVzVbVw8BqoHfZi22MKavcXFi5EtLTfV2SquWPP+Dll+G88+DUU+HOO2HjRhgxAhYvhh9/hIsugnHjoFkzJ0jEx/u61Md4EhQaAdvd3ie61rlbDQx0vb4GiBSRaNf6PiJSQ0TqAhcBTdz2G+uqcnpdRMIK+3ARuUNE4kQkLjk52YPiGmOK8/vv0KsXnHkmNGwIt98OS5eCqq9L5n9yc+GXX+CJJ6BdOzj9dHjsMTh6FJ59Ftasgc2b4dVX4YILnGDx6adOoPj732HGDGe/vn2doOHrfwPRYkogItcCl6vqba73NwLdVPUetzSnAhOBFsBinADRXlUPisgo4FogGdgD/Kqqb4pIQ2AXEApMAf5U1TFFlSU2NlZt7CNjSi8zE158EcaOhZo1nQvZunUwezYcPgytWsFNNzlLs2a+LWt2tlPe6tVBxDt5qkJKCmzdevySmOhc3ENDjy1hYce/L7iEhMBvv8HcuZCUBMHBTqDt39+5wDdt6lmZkpNh0iSYOBH27YNu3eCRR+Caa5w8vUFEVqhqrEdpPQgK3YHRqnq56/1IAFUdd5L0EcDvqtq4kG0fAh+o6pcF1vcCHlbVq4oqiwUFY46Xk5tDcnoySalJpGel06JOCxpGNEQKuYr+9JNzR7BhAwweDG+8AfXrO9vS0pxfr++9BwsXOut69YLhw2HgQIiIAFVlz+E9/Ln/T45kHSlROY8cgUMHQsg6HEFmWiRHD0Vw+EAEh/fXJGVfECkpzsV6375jfw8edPYNC4OoKGeJjj72t06UckpUJjXqpFG9diphkWmERKQhIUdI21+dA7sj2ZsUQXJiBElbI9m+JZRt25zg565GDWjSxLnIZ2Y6v/AzM49fjmbmQmhagSWV8JBwLoptyLVX1KfflWFERZXs389dejr85z/w2mvw55/Q8rRc7rh/Lz2vSuJg9i4ubHYh1UOqlypvbweFasAfwCXADmA5MFRV17ulqQukqGquiIwFclT1aVcj9Smquk9EOgEfAl1UNVtEGqpqkjjf3teBDFU9oWeTOwsKJlCkZ6WTlJrErrRdJKW5/rq9z1u35/AecjX3uH1rhtSkVVQrWke3pnVUaxpVb8V3s1rz+dTWNK5Tn8lvC1deWfjnqiq/bdzL5FmbmLt0E8nZmwmut4naLTeRUWMT6Tmp3j/YzJoE50RQLTeCUCIID4qkenAEEaERhASFcDjzMOnZaRzJTeWoppElaWQHp6LV0iA42+OPkdwQQomgenAEkaGR1KoeQVTNCKIiIqkZWpPMnExSj6aSlpmWv6RmOu/Ts4pveImqHkWDiAY0jGhIw8iGNKjZwPnrWtcgwnkfFhzG7sO7SUpNKvzfNjWJLXt3kZK5G4Jy8vP/aegGurduW6pT7NWg4MrwCuANIBiYpqpjRWQMEKeq80RkEE6PI8WpPrpbVY+KSDjwmyubQ8Bdeb2MROR7IAYQYJVrW5EdtSwomKos+XAyH6z5gGmrprFuz7oTtgdLMPUj6h+7wORdfFyvw6uFk7A/gU0pm9icsplNKZv4c18CORy7cEaGRh4XMJrUasLO1J1sStnkLPs2cfDowfz0QQRRM7s56YmtydndmjramstiW1GnRiQ7d8LOJNi5E3bvgpxj1y+CgqB+A6e94tRTneWU6ExCa6ZRrUYahKVCaBo5wWmkZ6c5F+Mst4ux6+KcmZNJRGgEkWGRRIRG5C+RoZHUDIkgPCiCoKxIco9GkJsRQfbhCHKOVifilCPUjEolvJYTRNzzzPucggEgrFrYcfkX/Lz8166y1AypSUZ2xvEX9sO78i/2SalJHM056tG/fZAEUa9mveMCSIOIhqTvbsDS+Q1ZuaQBK788k05ta5Tqu+X1oFBZWFAwBc2Jn8OH6z6k5Skt8y90raNbn7QKpbRyNZfUo6nUCqvl1Xyzc7P5evPXTFs1jXkb55Gdm805jc7h6jZX06hWo+N+YUZXjyY4yLNK5qQkuOce+HRONm26beUfozYRHLPpuIv/lgNbyNEcgiSIprWbOufOdf7y/jY/pTmhwaEcOQKff+5Ub3z7rVM3Hx0NLVseW1q0OPa6SROo5leztXifqnLw6MET7gQysjNOuHuoW6Mu1YJOfsJ27IBGBbv3lIAFBRMQDmYcpNWEVmTmZHIk6whZuVn522qE1HB+ERe40LWKakWDiAb5F/aj2UeLraJJSk1i9+HdZOdmc2rkqVzY7EIuaHoBFza7kHYx7QiSkg8MsGnfJt5d9S7vrX6Pnak7iakRw42dbuSWrrfQoV6HUp+T3Fx45x149FHIyIBnnoGHH3bqywvKyskiKS2J+jXrE1at0M5/hdq3z8mvVq3i05rKoSRBIcBjufFnL/34EnvT9xJ3exxdGnRh28Ft+b+C86pP1u5Zy9yNc8nOPVaFEhEaQcOIhuxN38v+jP0n5CsIMTVj8n/JdajXgQY1G3BK+Cms2r2KxVsXM3PdTADqhNfh/Kbn5weKMxueSUhwIVdg4HDmYWZvmM20VdNYvHUxQRJEn1Z9mNBnAle1uYpgQvn2W3hhutPQWFjjamF/a9d2qmt+/x3uuAOWLHH6wf/rX9C69cnPX0hwCE1re9hFxk10dIl3MX7E7hSMX0o8lEjrCa0Z0HYAMwbMKDJtdm42Ww9szQ8Ym1I2sSttFzE1YgptCKxXs16Rt/Kqyl8H/mLJ1iUs3rqYJduWsCllE+DcoXRv3D0/SJzT+BzW7F7DtJXTmLluJqmZqbSKasWtXW7l5i43c2rkqWzY4PT6ef99p9onKgrOOgsOHDjWE+fAgZMfX1AQ1KnjDKMQEQHjx8Mtt3ivG6fxf1Z9ZKq8W+feyoy1M9g4YiPNT2nu6+KQlJrE0m1L84PEmt1rUBRBUJQaITW4rv113NrlVs5vej4pKcLMmU4dfVycU/9+xRVw881w5ZVON0x3OTmwf/+J3Tbdu3KGhzvVRnndTI3JY0HBVGlrd6+l8+TOPNj9QcZfNt7XxSnUgYwD/LjtR5YlLqP5Kc25rv11VA+qxVdfOXcFX3wBWVnQpYsTCIYOhXr1fF1qU1VZUDBV2hUzrmBZ4jL+vPdPoqqX4WmhCrJqlRMIZsxwnl6tVw+GDXOCQefOvi6dCQTW0GyqrAUJC/hq81e8cukrlTIgHDniBIG4OGf5+WdngLTQULj6aicQ9O5deG8gYyoDCwrGb+RqLo98+wjNajdjRLcRvi4OmZnOYGd5ASAuzhlHKO8hrgYNIDYW7r3XGVbCeu0Yf2BBwfiNj9Z+xMpdK/ngmg8IrxZeprwyM506/ZwcZ+C1nJzil4yM44PAmjVOPuBc8M8+27kbiI11lrI8bGSMr1hQMH4hIzuDUd+PomuDrgzpOKTU+WRmOsMav/WW86BXadSu7Vz0H3jgWABo1sy6gJqqwYKC8QsTf53I1oNbmdp3aqmeIAZniOTrroNff3VG/2zXzhmauLilWjXnb0gItG0Lp53mPBtgTFVkQcFUeilHUhi7ZCx9WvXhkpaXlCqP//7XmSMgJwc++QQGDfJyIY2pIiwoBIiUIync9d+7+N+m/9H8lOb54wC5jwvUqFajUv8KL08vLHmBgxkHeelvL5V43+xsePJJeOkl55mATz5xJpIxxhTOgkIAWLRlETd8dgO7D+/mpk43se/IPjalbGL+5vnHDe1bvVp1Tos6LX8QubwhltvFtKNeTd88WbXlwBYm/DqB4V2G07F+xxLtu2OH0+tn6VJnntw33nCe+jXGnJwFhSosKyeLZ394lheWvECrqFYs+/syYk899vxKruaSeCgxfzygvL/xe+P536b/kZnjdK0JlmBW3bWqTKN3ltaT3z9JsAQz5qIiZ2o9wTffOA+IHTniPDQ2dGg5FdCYKsaCQhWVsD+BYZ8N4+fEn7mlyy281ectIkIjjkuTN45+09pNT6irz8nNYdvBbWxI3kD/j/vz/ur3eenSklfflMWKnSuYsXYGT5z/BI1rnTC7a6FycmDMGHjuOachefZsOOOMci6oMVVI5atANmX24doP6TK5C/HJ8cwcOJNp/aadEBCKExwUTIs6LbiyzZVcftrlfLTuoxOmfSxPqsoj3z5C3Rp1ebTHox7ts2sXXHaZExSGD3d6GVlAMKZkPAoKItJbRDaKyGYROWEeZRFpJiILRGSNiCwSkcZu214SkXWu5Xq39S1E5BcR2SQiH4tIqHcOKXAdOnqIm+bcxLDPhtGpfidW3bWK6ztcX/yOxRjWcRjbD21n6balXiilZ+Zvns/CLQt5+sKnqR1eu9j0ixZB166wbBlMm+YsNUo3c6ExAa3YoCAiwcAkoA/QDhgiIu0KJBsPTFfVTsAYnPmaEZErgTOBLsA5wCMikjdf00vA66raGtgP/L3shxO4ft3xK13/1ZUZa2cwuudoFg1f5LUhpfue3pcaITX4cO2HXsmvODm5OTz63aO0imrFnbF3Fpk2NxdeeAEuucR5qOzXX525BIwxpePJnUI3YLOqJqhqJjAT6FcgTTtggev1Qrft7YAfVDVbVQ8Dq4He4syFeDEw25XuPaB/6Q8jcOXk5jBuyTh6TOtBdm42Pwz/gWd6PVPkJDElVTO0Jv3P6M8nGz7Jb3wuT++tfo91e9Yx7pJxhAYXfQP56KMwapTTyyguDjpUfFu4MVWKJ0GhEbDd7X2ia5271cBA1+trgEgRiXat7yMiNUSkLnAR0ASIBg6oanYReZpi7Di0g0vfv5Qnvn+CAW0HsPqu1Zzf9Pxy+axhHYeRciSFrzd/XS7550nPSuephU9xTqNzGNh2YJFpk5Jg4kRn5NEPPnBmHTPGlI0nQaGwEV0KTsLwMNBTRFYCPYEdQLaqfgN8CfwEfAQsA7I9zNP5cJE7RCROROKSk5M9KG5gmL95Pp0md+LXHb8yre80Zg6cySnhp5Tb513a8lKiq0fz4bryrUJ64+c32Jm6k/GXjUeKGUzo1Vedh9OeftrGHTLGWzwJCok4v+7zNAZ2uidQ1Z2qOkBVuwKjXOsOuv6OVdUuqnopTjDYBOwFThGRaifL0y3vKaoaq6qxMTExJTi0qutAxgGGfjqURpGN+O3O37il6y3FXkDLKiQ4hOvaX8fc3+eSejS1XD4j+XAyLy59kf5n9C/2jmfvXnj7bef5g5Yty6U4xgQkT4LCcqC1q7dQKDAYmOeeQETqiuSPjzASmOZaH+yqRkJEOgGdgG/Ume5tIZA3As3NwNyyHkygeG3Za+zP2M/0a6bTJrpNhX3usI7DOJJ9hLkby+ef6rnFz5Gelc64S8YVm/aNN5wH00aOLJeiGBOwig0Krnr/EcDXQDwwS1XXi8gYEenrStYL2CgifwD1gbGu9SHAEhHZAEwBbnBrR3gMeFBENuO0MUz10jFVacmHk3n959e5tt21dGnQpUI/u3uT7jSr3axceiHtTN3Jv1b8i1u73soZdYt+uODAAZgwwRnUrm1brxfFmIDmURcVVf0Sp23Afd3Tbq9nc6wnkXuaDJweSIXlmYDTs8mUwItLXyQ9K73Ewz54Q5AEMaTDEF756RX2HN7j1fGQXlv2Gtm52Tx+/gmPwZxg4kQ4dMjpdWSM8S57otmP7Di0g0nLJ3FT55uK/TVdXoZ1GkaO5vDJ+k+8lmfKkRQmx01mcIfBtKxTdANBWhq8/rozw5lNem+M91lQ8CPPL36eXM3lmZ7P+KwMHep1oGO9jl7thTTx14kczjrM4z2Kv0uYPBlSUuwuwZjyYkHBTyTsT+Cdle9w+5m3e+1J5dIa2nEoP23/ib/2/1XmvA5nHuatX97i6jZXFzs09pEjMH48XHopnHNOmT/aGFMICwp+YvSi0VQLqsaoC33/E3lIB2eO5I/WfVTmvP7927/Zd2QfI88vvhvR1Kmwe7czaY4xpnxYUPADG5I38MGaD7in2z2cGnmqr4tDs1OacX7T85mxdgZO7+LSyczJ5NVlr9KzWU+6N+ledNpMZ/a0Cy6ACy8s9UcaY4phQcEPPL3waSJCI3isx2O+Lkq+oR2GsiF5A2t2ryl1Hh+s+YDEQ4ke3SVMnw6JiXaXYEx5s6BQya3YuYJP4z/lwe4PEl0j2tfFyXdt+2upFlSt1M8s5OTm8OLSF+naoCuXnXZZkWmzs2HcODj7bKc9wRhTfiwoVHJPLnySqOpRPNj9QV8X5Th1a9Qt0+Q7n8V/xqaUTYw8f2SxQ3TMnAkJCU6PIxvjyJjyZUGhElu6bSnzN8/n8R6PUyusVvE7VLChHYeWavIdVWXc0nG0iW7DgLYDikybmwtjx0LHjs6zCcaY8mVBoZJSVUZ9P4oGEQ24u9vdvi5Oofqd3q9Uk+988+c3rNy1ksd6PEZwUHCRaT/7DH7/3blLCLJvqzHlzv6bVVLfJnzL4q2LefKCJ6kRUjnnlSzt5Dvjlo6jca3G3NDphiLTqcLzz0ObNs44R8aY8mdBoRLKu0tofkpzbj/rdl8Xp0hDOwwt0eQ7y7Yv44etP/BQ94eKnVXtf/+D1avhiScguOgbCmOMl1hQqIQ+//1z4nbG8UzPZ4q9cPraZaddVqLJd8YtHUd09WhuP7PoYJd3l9C8uTNngjGmYlhQOIll25dx31f3sePQjgr93JzcHJ5a+BSnR59ebPVKZeA++U5aZlqRadfuXssXf3zBvefcS83QmkWmXbAAfvkFHn8cQkK8WWJjTFEsKBQiPjmeKz68grd+fYu2k9oy8deJ5OTmVMhnz1w3k/XJ6xlz0RiqBXk0srnPDe04lCPZR/j898+LTPfijy8SERrBiG4jis3z+efh1FNh+HAvFdIY4xELCgXsTtvNFR9eQVhwGN/f9D3dm3Tnnq/uofvU7qzatapcPzsrJ4unFz1NlwZdGNTOf1pWz2tyXrGT7yTsT2DmupncedadRFWPKjK/JUvghx/g0UchLMzbpTXGFMWCgpv0rHSu/uhq9hzew3+H/peLWlzE/GHz+WjgR2w7uI3YKbE89PVDxVaTlNa7q94lYX8Cz1/0PEHiP/80eZPvfPPnN+w5vKfQNK/8+ArVgqp59BDe2LEQEwO3V+42dmOqJP+58pSznNwchn02jLidcXw08CNiT40FQEQY3GEw8XfHc9uZt/Haz6/RblI7vtj4hVc/PyM7g+cWP0f3xt25ovUVXs27IgztOPSkk+/sStvFu6veZXjn4cUO6Ld8OXz9NTz0ENSonD1xjanSPAoKItJbRDaKyGYROWEmFBFpJiILRGSNiCwSkcZu214WkfUiEi8ib4lrTANXuo0issq1eG9ux1J4+JuH+fz3z3mz95v0Pb3vCdvrVK/D5Ksm8+OtP1I7vDZ9Z/ZlwMcDSDyU6JXPnxw3mcRDiYy9eGyxwz5URh3rdzzp5DuvL3udrNwsHu3xaLH5jB0LderAP/5RHqU0xhRLVYtcgGDgT6AlEAqsBtoVSPMJcLPr9cXA+67X5wE/uvIIBpYBvVzbFgGxxX2++3LWWWdpeXjz5zeV0ej9X93vUfrM7Ex9ccmLWv356hrxQoS++fObmp2TXerPTz2aqjEvx+jfpv+t1HlUBuOWjFNGowkpCfnrUtJTNOKFCB08e3Cx+69erQqqo0eXZymNCTxAnHp4nfXkTqEbsFlVE1Q1E5gJ9CuQph2wwPV6odt2BcJdwSQMCAF2exivKsTc3+dy//z76X9Gf8ZfNt6jfUKCQ3js/MdY93/r6NGkB/fNv49zp57Lb0m/lfjzVZU3fn6D5PRkxl48tsT7VyaFTb4zafkk0jLTCp1q88ABp7roww/h2WednkaRkXDPPRVVYmNMQZ70eWwEbHd7nwgUnAxxNTAQeBO4BogUkWhVXSYiC4EkQICJqhrvtt+7IpIDfAo874poFWb5juUM+XQIsafGMmPAjGLH4SmoZZ2WfDXsK2atn8V98+/j7H+fzZ1n3Umz2s1IzUwlLTMtfznh/dFj7xWl7+l96daoWzkdacVwn3xn5PkjOZJ9hDd+fpMe9a7g9x8688Um2OS27N17/P5NmsBrr0FU0Z2TjDHlyJOgUFgFd8GL98PARBEZDiwGdgDZItIKaAvktTF8KyIXqupiYJiq7hCRSJygcCMw/YQPF7kDuAOgadOmHhTXM1sObOHqj66mfkR9vhjyRanHFxIRru9wPZe3upyR341kctxkFCVIgogIjchfIkMjiQiNoGFEQyLDIokIca0Pi6R2WG2GdxnutWPzpaEdhvJ/X/4fPQasYV3aD6Sev5cfJ43kx23O9kaNoHVruOYa52/r1tCqFZx2GlSv7tuyG2NAivtxLiLdgdGqernr/UgAVR13kvQRwO+q2lhEHgHCVfU517angQxVfbnAPsNx2heKfKopNjZW4+LiPDqwouw/sp8e03qQlJbET7f+RNuYtmXOM0/q0VSCg4KpXq26XzYYl8Xy5fDw03tZfHZDIjaMQNt+SkxIM17tsIRWrZyLv/UoMqbiicgKVY31JK0nbQrLgdYi0kJEQoHBwLwCH1hXJL9j/Uhgmuv1NqCniFQTkRCgJxDvel/XtW8IcBWwzpMCl1VmTiYDZg1gc8pm5lw/x6sBASAyLJIaITUCKiDEx8PAgdCtG2yIq0u7sMtJ7/QWh6tt559DnmDAAOjUyQKCMf6g2KCgqtnACOBrIB6YparrRWSMiOT13ewFbBSRP4D6QF6L6WycnktrcdodVqvqFziNzl+LyBpgFU5107+9dlQnPxZum3cbi7YsYlq/afRq3qu8P7JK27YNbr0VOnSAb7+F0aNdM6T1HUqu5tKlQRd6t+rt62IaY0rAo8F1VPVL4MsC6552ez0bJwAU3C8HuLOQ9YeBs0pa2LJ69odneX/N+4zpNcYvBpurrPbsgRdegLffdqbHvO8+GDnSeQoZnMl3ejTpwVMXPhVQd0zGVAX+MeKaF7y36j2e/eFZbulyC09e+KSvi+OXDh2CV191egilp8Mtt8Azzzi9htzVDK3J0ltLNkWnMaZyCIigsCBhAbd9cRt/a/k3/nXVv+zXawllZMA//+ncHezb58yC9txzcMYZvi6ZMcbbqnxQUFXGLhnL6dGnM/va2YQE2+D8JbFggfNQWWIiXHaZExjOqvCKP2NMRanyQUFEmDdkHgczDlI7vLavi+M3VGH8eGeSmzPOgO+/h4su8nWpjDHlrcoHBSD/ATLjmbQ0p1fRJ5/AtdfCtGkQYafPmIBgQ2eb42zaBOeeC59+Ci+/DB9/bAHBmEASEHcKxjP//S/ccANUq+bMafC3v/m6RMaYimZ3CobcXGeU0quvhpYtIS7OAoIxgcruFALcwYNw443wxRdw000webINTGdMILOgEMDWr3dGK/3rL5gwAe6+23lC2RgTuCwoBKjZs53nDyIinO6mF1zg6xIZYyoDa1MIMDk5zrMH114LHTvCihUWEIwxx9idQgDZtw+GDHFGNL3rLnjjDQgL83WpjDGViQWFAPHnn9CnD2zdClOnOg+nGWNMQRYUAsAvvzjdTXNznfaDHj18XSJjTGVlbQpV3Ny5zphFkZHw008WEIwxRbOgUIVNmOB0Oe3YEZYtgzZtfF0iY0xlZ0GhCsrNhYcfhnvvhb59YeFCqFfP16UyxvgDj4KCiPQWkY0isllEHi9kezMRWSAia0RkkYg0dtv2soisF5F4EXlLXDPciMhZIrLWlWf+elM2GRkweLAzQ9rddzsD29Wo4etSGWP8RbFBQUSCgUlAH6AdMERE2hVINh6YrqqdgDHAONe+5wE9gE5AB+BsoKdrn7eBO4DWrsVmeC+jffucMYs++cSZC2HCBAgO9nWpjDH+xJM7hW7AZlVNUNVMYCbQr0CadsAC1+uFbtsVCAdCgTAgBNgtIg2BWqq6TFUVmA70L9ORBLiEBKcROS7OGe76oYdsyApjTMl5EhQaAdvd3ie61rlbDQx0vb4GiBSRaFVdhhMkklzL16oa79o/sZg8ARCRO0QkTkTikpOTPShu4Fm+HLp3h+Rk+O47uO46X5fIGOOvPAkKhf3e1ALvHwZ6ishKnOqhHUC2iLQC2gKNcS76F4vIhR7m6axUnaKqsaoaGxMT40FxA8u8edCzJ9Ss6XQ5Pf98X5fIGOPPPAkKiUATt/eNgZ3uCVR1p6oOUNWuwCjXuoM4dw0/q2qaqqYBXwHnuvJsXFSepnj//KfT5bR9e6fL6emn+7pExhh/50lQWA60FpEWIhIKDAbmuScQkboikpfXSGCa6/U2nDuIaiISgnMXEa+qSUCqiJzr6nV0EzDXC8cTMJ591ulddMUVsGgR1K/v6xIZY6qCYoOCqmYDI4CvgXhglqquF5ExItLXlawXsFFE/gDqA2Nd62cDfwJrcdodVqvqF65t/wD5g9+NAAAVNklEQVTeATa70nzllSMKAHPnwujRcPPNMGeOU3VkjDHeIE7nH/8QGxurcXFxvi6GTyUkwJlnQuvWsHSpjXJqjCmeiKxQ1VhP0toTzX7k6FGnZ5EIzJplAcEY4302SqofeeghZ1KcuXOhRQtfl8YYUxXZnYKf+PhjmDTJGdOob9/i0xtjTGlYUPADf/wBt90G550HL7zg69IYY6oyCwqV3JEjznzKYWEwcyaEhPi6RMaYqszaFCq5e++FNWvgq6+gSZPi0xtjTFnYnUIlNn06vPMOjBoFvW0MWWNMBbCgUEmtXw//+Af06uU8qGaMMRXBgkIllJbmtCNERsKHH0I1q+QzxlQQu9xUMqpw112wcSN8+y00bOjrEhljAokFhUrmnXdgxgwYMwYuvtjXpTHGBBqrPqpEVq2Ce+6Byy5zGpeNMaaiWVCoJA4dctoR6taFDz6AIPuXMcb4gFUfVQKqzhPLf/3lzI1gE8wZY3zFgkIlMGkSfPIJvPSSTadpjPEtq6TwsRkz4IEH4KqrnMHujDHGlywo+NCECXDDDXDBBU5wsHYEY4yveXQZEpHeIrJRRDaLyOOFbG8mIgtEZI2ILBKRxq71F4nIKrclQ0T6u7b9R0T+ctvWxbuHVnmpOk8p33sv9O8PX34JtWr5ulTGGONBm4KIBAOTgEuBRGC5iMxT1Q1uycYD01X1PRG5GBgH3KiqC4EurnyicOZj/sZtv0dUdbZ3DsU/5ObCfffBxIlwyy0wZYo9sWyMqTw8uVPoBmxW1QRVzQRmAv0KpGkHLHC9XljIdoBBwFeqml7awvq7rCy48UYnIDz0EEydagHBGFO5eBIUGgHb3d4nuta5Ww0MdL2+BogUkegCaQYDHxVYN9ZV5fS6iFTpGYfT052qog8/hHHj4JVXnLmWjTGmMvEkKBR26dIC7x8GeorISqAnsAPIzs9ApCHQEfjabZ+RwBnA2UAU8FihHy5yh4jEiUhccnKyB8WtfA4cgMsvd+ZE+Ne/4PHHLSAYYyonT4JCIuA+vUtjYKd7AlXdqaoDVLUrMMq17qBbkuuAOaqa5bZPkjqOAu/iVFOdQFWnqGqsqsbG+OFTXbt2Qc+e8MsvMGsW3HGHr0tkjDEn50lQWA60FpEWIhKKUw00zz2BiNQVkby8RgLTCuQxhAJVR667B0REgP7AupIXv3L76y/nYbQ//4T//Q8GDfJ1iYwxpmjFBgVVzQZG4FT9xAOzVHW9iIwRkb6uZL2AjSLyB1AfGJu3v4g0x7nT+KFA1jNEZC2wFqgLPF+mI6lk1q2DHj1g/35YsAAuvdTXJTLGmOKJasHmgcorNjZW4+LifF2MYi1bBldeCdWrwzffQPv2vi6RMSaQicgKVY31JK09Q+tlX38Nf/sbREfDjz9aQDDG+BcLCl60ejVcfTW0aQNLl0Lz5r4ukTHGlIw9OuVFL78M4eFOG0JUlK9LY4wxJWd3Cl6ybRt8/DHceacFBGOM/7Kg4CVvvuk8kHbvvb4uiTHGlJ4FBS84cMAZ2O7666FJk+LTG2NMZWVBwQv+/W9IS3MGuTPGGH9mQaGMMjOdqqNLLoGuXX1dGmOMKRvrfVRGH38MO3bAO+/4uiTGGFN2dqdQBqowfjx06OCMgmqMMf7O7hTK4LvvYM0aePddGwrbGFM12J1CGbz6KjRsCEOG+LokxhjjHRYUSmnNGmeco3vugbAqPWecMSaQWFAopddeg5o1nSeYjTGmqrCgUAo7djhzLf/97zakhTGmarGgUAoTJkBODtx/v69LYowx3mVBoYRSU2HyZGdqzRYtfF0aY4zxLgsKJTR1Khw8CA8/7OuSGGOM93kUFESkt4hsFJHNIvJ4IdubicgCEVkjIotEpLFr/UUissptyRCR/q5tLUTkFxHZJCIfi0iodw/N+7Kz4fXX4YIL4OyzfV0aY4zxvmKDgogEA5OAPkA7YIiItCuQbDwwXVU7AWOAcQCqulBVu6hqF+BiIB34xrXPS8Drqtoa2A/83QvHU65mz3bmTbC7BGNMVeXJnUI3YLOqJqhqJjAT6FcgTTtggev1wkK2AwwCvlLVdBERnCAx27XtPaB/SQtfkfKGtGjTBq66ytelMcaY8uFJUGgEbHd7n+ha5241MND1+hogUkSiC6QZDHzkeh0NHFDV7CLyrFQWL4YVK5zhsYOsJcYYU0V5cnkrbFQfLfD+YaCniKwEegI7gLwLPiLSEOgIfF2CPPP2vUNE4kQkLjk52YPilo/x4yEmBm680WdFMMaYcudJUEgE3OcTawzsdE+gqjtVdYCqdgVGudYddEtyHTBHVbNc7/cCp4hI3oB8J+TplvcUVY1V1diYmBgPiut98fHw3//CiBFQvbpPimCMMRXCk6CwHGjt6i0UilMNNM89gYjUFZG8vEYC0wrkMYRjVUeoquK0PQxyrboZmFvy4leM116D8HD4xz98XRJjjClfxQYFV73/CJyqn3hglqquF5ExItLXlawXsFFE/gDqA2Pz9heR5jh3Gj8UyPox4EER2YzTxjC1TEdSTnbtgunTYfhwp/rIGGOqMo/mU1DVL4EvC6x72u31bI71JCq47xYKaURW1QScnk2V2qRJkJUFDzzg65IYY0z5s340RTh8GP75T+jXz+mKaowxVZ0FhSL85z+QkmIPqxljAocFhZPIyXGGtDj3XDjvPF+XxhhjKoYFhZOYNw/+/NO5S7D5l40xgcKCwknMnQvR0dC/Ug++YYwx3mVB4SQWLYJevSA42NclMcaYimNBoRBbtsDWrU5QMMaYQGJBoRCLFjl/LSgYYwKNBYVCLFoEdetCu4KzRhhjTBVnQaEQixZBz542RLYxJvDYZa8Aa08wxgQyCwoFWHuCMSaQWVAowNoTjDGBzIJCAdaeYIwJZHbpc2PtCcaYQGdBwY21JxhjAp0FBTfWnmCMCXQWFNxYe4IxJtB5dPkTkd4islFENovI44VsbyYiC0RkjYgsEpHGbtuaisg3IhIvIhtcczYjIv8Rkb9EZJVr6eKtgyoNa08wxhgPgoKIBAOTgD5AO2CIiBSsYBkPTFfVTsAYYJzbtunAK6raFmdO5j1u2x5R1S6uZVUZjqPMrD3BGGM8u1PoBmxW1QRVzQRmAv0KpGkHLHC9Xpi33RU8qqnqtwCqmqaq6V4puZdZe4IxxngWFBoB293eJ7rWuVsNDHS9vgaIFJFooA1wQEQ+E5GVIvKK684jz1hXldPrIhJW2IeLyB0iEiciccnJyR4dVGlYe4IxxngWFAqbjFILvH8Y6CkiK4GewA4gG6gGXODafjbQEhju2mckcIZrfRTwWGEfrqpTVDVWVWNjYmI8KG7JWXuCMcY4PAkKiUATt/eNgZ3uCVR1p6oOUNWuwCjXuoOufVe6qp6ygc+BM13bk9RxFHgXp5rKJ6w9wRhjHJ4EheVAaxFpISKhwGBgnnsCEakrInl5jQSmue1bR0TyfuJfDGxw7dPQ9VeA/sC6shxIWVh7gjHGOIoNCq5f+COAr4F4YJaqrheRMSLS15WsF7BRRP4A6gNjXfvm4FQdLRCRtThVUf927TPDtW4tUBd43mtHVULWnmCMMQ5RLdg8UHnFxsZqXFycV/PcsgVatIAJE2DECK9mbYwxlYKIrFDVWE/SBvxvY2tPMMaYY6r5ugC+Zu0JxviPrKwsEhMTycjI8HVRKqXw8HAaN25MSEhIqfOwoLDI2hOM8ReJiYlERkbSvHlznD4qJo+qsm/fPhITE2nRokWp8wnoS+Fff9nzCcb4k4yMDKKjoy0gFEJEiI6OLvNdVEAHBWtPMMb/WEA4OW+cm4APCtaeYIwxxwRsUFC19gRjjCkoYC+HW7bAtm1WdWSMKZn+/ftz1lln0b59e6ZMmQLA/PnzOfPMM+ncuTOXXHIJAGlpadxyyy107NiRTp068emnn/qy2B4L2N5H1p5gjH+7/35Y5eVZWLp0gTfeKDrNtGnTiIqK4siRI5x99tn069eP22+/ncWLF9OiRQtSUlIAeO6556hduzZr164FYP/+/d4tbDkJ6KBg7QnGmJJ66623mDNnDgDbt29nypQpXHjhhfndQKOiogD47rvvmDlzZv5+derUqfjClkJABgVrTzDG/xX3i748LFq0iO+++45ly5ZRo0YNevXqRefOndm4ceMJaVXVL3tKBeQl0doTjDGlcfDgQerUqUONGjX4/fff+fnnnzl69Cg//PADf/31F0B+9dFll13GxIkT8/f1l+qjgAwK1p5gjCmN3r17k52dTadOnXjqqac499xziYmJYcqUKQwYMIDOnTtz/fXXA/Dkk0+yf/9+OnToQOfOnVm4cKGPS++ZgKw+svYEY0xphIWF8dVXXxW6rU+fPse9j4iI4L333quIYnlVwN0pWHuCMcacXMBdFq09wRhjTi7ggoK1JxhjzMkFZFCw9gRjjCmcR0FBRHqLyEYR2SwijxeyvZmILBCRNSKySEQau21rKiLfiEi8iGwQkeau9S1E5BcR2SQiH4tIqLcO6mSsPcEYY4pW7KVRRIKBSUAfoB0wREQK/s4eD0xX1U7AGGCc27bpwCuq2hboBuxxrX8JeF1VWwP7gb+X5UA8Ye0JxhhTNE9+L3cDNqtqgqpmAjOBfgXStAMWuF4vzNvuCh7VVPVbAFVNU9V0cR7zuxiY7drnPaB/mY7EA9aeYIwxRfMkKDQCtru9T3Stc7caGOh6fQ0QKSLRQBvggIh8JiIrReQV151HNHBAVbOLyBMAEblDROJEJC45OdmzozqJvPaE9u3LlI0xxngkIiLC10UoMU+CQmGDd2iB9w8DPUVkJdAT2AFk4zwcd4Fr+9lAS2C4h3k6K1WnqGqsqsbGxMR4UNzC5bUn9OoFfjgciTHGVAhPnmhOBJq4vW8M7HRPoKo7gQEAIhIBDFTVgyKSCKxU1QTXts+Bc4FpwCkiUs11t3BCnt6W157w6KPl+SnGmIpy//z7WbXLu2Nnd2nQhTd6n3ykvccee4xmzZrxf//3fwCMHj0aEWHx4sXs37+frKwsnn/+efr1K1jDfqK0tDT69etX6H7Tp09n/PjxiAidOnXi/fffZ/fu3dx1110kJCQA8Pbbb3Peeed54aiP50lQWA60FpEWOHcAg4Gh7glEpC6Qoqq5wEici37evnVEJEZVk3HaEeJUVUVkITAIp43iZmCuNw7oZKw9wRhTVoMHD+b+++/PDwqzZs1i/vz5PPDAA9SqVYu9e/dy7rnn0rdv32JHSA0PD2fOnDkn7LdhwwbGjh3Ljz/+SN26dfMH2Lv33nvp2bMnc+bMIScnh7S0tHI5xmKDgqpmi8gI4GsgGJimqutFZAzOBX4e0AsYJyIKLAbudu2bIyIPAwtcjcsrgH+7sn4MmCkizwMrganePbTj2fMJxlQtRf2iLy9du3Zlz5497Ny5k+TkZOrUqUPDhg154IEHWLx4MUFBQezYsYPdu3fToEGDIvNSVZ544okT9vv+++8ZNGgQdevWBY7Nz/D9998zffp0AIKDg6ldu3a5HKNHA+Kp6pfAlwXWPe32ejbHehIV3PdboFMh6xNwejaVO2tPMMZ4y6BBg5g9eza7du1i8ODBzJgxg+TkZFasWEFISAjNmzcnIyOj2HxOtp+v52EIiEe47PkEY4y3DB48mJkzZzJ79mwGDRrEwYMHqVevHiEhISxcuJCtW7d6lM/J9rvkkkuYNWsW+/btA47Nz3DJJZfw9ttvA5CTk8OhQ4fK4egCJChYe4Ixxlvat29PamoqjRo1omHDhgwbNoy4uDhiY2OZMWMGZ5xxhkf5nGy/9u3bM2rUKHr27Ennzp158MEHAXjzzTdZuHAhHTt25KyzzmL9+vXlcnyiWmhP0EopNjZW4+LiSrzfzTfDl1/Cnj1WfWSMP4uPj6dt27a+LkalVtg5EpEVqhrryf4BMclO27Zw6qkWEIwxpjgBERQeP2EIP2OMqRhr167lxhtvPG5dWFgYv/zyi49KVLSACArGGOMrHTt2ZNUq7z5kV54CoqHZGFN1+FM7aEXzxrmxoGCM8Rvh4eHs27fPAkMhVJV9+/YRHh5epnys+sgY4zcaN25MYmIiZR0xuaoKDw+ncePGxScsggUFY4zfCAkJoUWLFr4uRpVm1UfGGGPyWVAwxhiTz4KCMcaYfH41zIWIJANbgbrAXh8XpzKw8+Cw8+Cw8+Cw83BM3rlopqoeTV3pV0Ehj4jEeTqOR1Vm58Fh58Fh58Fh5+GY0pwLqz4yxhiTz4KCMcaYfP4aFKb4ugCVhJ0Hh50Hh50Hh52HY0p8LvyyTcEYY0z58Nc7BWOMMeXAr4KCiPQWkY0isllEAnqWBBHZIiJrRWSViJR8Ojo/JSLTRGSPiKxzWxclIt+KyCbX3zq+LGNFOMl5GC0iO1zfiVUicoUvy1gRRKSJiCwUkXgRWS8i97nWB9R3oojzUOLvhN9UH4lIMPAHcCmQCCwHhqjqBp8WzEdEZAsQq6oB1R9bRC4E0oDpqtrBte5lIEVVX3T9WKijqo/5spzl7STnYTSQpqrjfVm2iiQiDYGGqvqbiEQCK4D+wHAC6DtRxHm4jhJ+J/zpTqEbsFlVE1Q1E5gJ9PNxmUwFU9XFQEqB1f2A91yv38P5z1ClneQ8BBxVTVLV31yvU4F4oBEB9p0o4jyUmD8FhUbAdrf3iZTyoKsIBb4RkRUicoevC+Nj9VU1CZz/HEA9H5fHl0aIyBpX9VKVrjIpSESaA12BXwjg70SB8wAl/E74U1CQQtb5R91X+eihqmcCfYC7XdUJJrC9DZwGdAGSgFd9W5yKIyIRwKfA/ap6yNfl8ZVCzkOJvxP+FBQSgSZu7xsDO31UFp9T1Z2uv3uAOTjVa4Fqt6tONa9udY+Py+MTqrpbVXNUNRf4NwHynRCREJwL4QxV/cy1OuC+E4Wdh9J8J/wpKCwHWotICxEJBQYD83xcJp8QkZquxiREpCZwGbCu6L2qtHnAza7XNwNzfVgWn8m7CLpcQwB8J0REgKlAvKq+5rYpoL4TJzsPpflO+E3vIwBXd6o3gGBgmqqO9XGRfEJEWuLcHYAze96HgXIuROQjoBfO6I+7gWeAz4FZQFNgG3CtqlbpRtiTnIdeONUECmwB7syrV6+qROR8YAmwFsh1rX4Cpz49YL4TRZyHIZTwO+FXQcEYY0z58qfqI2OMMeXMgoIxxph8FhSMMcbks6BgjDEmnwUFY4wx+SwoGGOMyWdBwRhjTD4LCsYYY/L9P+NYn/W+K+UDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8a6fac5438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VOW9+PHPdyYLSdhDIJCwhEW2REBCqBvWFey1cFVUlKvCtbXWuhcq1h2tWq1L7738XGpFbaVCsd5yBcEFW7SyBQxrWEIIEMKShT1Akpnn98czkRBCMklmcpKZ7/v1Oq+cOdt85zB8zzPPec7ziDEGpZRS4cPldABKKaWaliZ+pZQKM5r4lVIqzGjiV0qpMKOJXymlwowmfqWUCjOa+JVSKsxo4ldKqTCjiV8ppcJMhNMBVNepUyfTq1cvp8NQSqkWZdWqVUXGmAR/tm12ib9Xr15kZmY6HYZSSrUoIrLD3221qkcppcKMJn6llAozmviVUirMNLs6fqVUeCovLyc/P58TJ044HUqz1qpVK5KTk4mMjGzwMTTxK6Wahfz8fNq0aUOvXr0QEafDaZaMMRQXF5Ofn09KSkqDj6NVPUqpZuHEiRPEx8dr0q+FiBAfH9/oX0Wa+JVSzYYm/boF4hyFTuIvKYFnnoFVq5yORCmlmrXQqeN3u+HJJ+388OHOxqKUapFat27N0aNHnQ4j6EKnxN+uHQwYAMuXOx2JUko1a6GT+AFGjoQVK8AYpyNRSrVgxhimTp1KamoqaWlpzJ49G4A9e/YwatQohg4dSmpqKl9//TUej4dJkyZ9v+2rr77qcPR1C52qHoCMDHj3XcjLg0Y0dVJKOeyBByArK7DHHDoUXnvNr03/9re/kZWVxZo1aygqKmLEiBGMGjWKWbNmMXr0aB599FE8Hg+lpaVkZWWxe/du1q9fD8DBgwcDG3cQhF6JH2ypXymlGuibb77h5ptvxu1206VLFy655BJWrlzJiBEjmDlzJk899RTr1q2jTZs29O7dm9zcXO69914WLlxI27ZtnQ6/TqFV4k9Lg1atbD3/TTc5HY1SqqH8LJkHizlLdfGoUaNYsmQJ8+fP59Zbb2Xq1KncdtttrFmzhkWLFjFjxgzmzJnDO++808QR109olfgjI+G887TEr5RqlFGjRjF79mw8Hg+FhYUsWbKEjIwMduzYQefOnfnpT3/KHXfcwerVqykqKsLr9XL99dfzzDPPsHr1aqfDr1NolfjB1vO/8QaUl9sLgVJK1dO1117L0qVLGTJkCCLCiy++SGJiIu+99x4vvfQSkZGRtG7dmvfff5/du3czefJkvF4vAM8//7zD0ddNzvaT5rSNRMYAvwfcwNvGmBeqrR8FvAacC0wwxsyttr4tkA18bIy5p7b3Sk9PN40aiOXDD+Hmm2H1ahg2rOHHUUo1qezsbAYOHOh0GC1CTedKRFYZY9L92b/Oqh4RcQMzgKuBQcDNIjKo2mY7gUnArLMc5hngn/4E1GgZGfavVvcopVSN/KnjzwByjDG5xpgy4ENgXNUNjDF5xpi1gLf6ziIyHOgCfBaAeOuWkgKdOumDXEopdRb+JP4kYFeV1/m+ZXUSERfwMjC1ju3uFJFMEcksLCz059C1HcyW+rXEr5RSNfIn8dfUFZy/j8beDSwwxuyqbSNjzFvGmHRjTHpCgl+DxNdu5EjYuBEOH278sZRSKsT406onH+he5XUyUODn8c8HLhaRu4HWQJSIHDXGTKtfmPWUkWG7bVi1Ci69NKhvpZRSLY0/Jf6VQD8RSRGRKGACMM+fgxtjJhpjehhjegFTgPeDnvTh1A1eredXSqkz1Jn4jTEVwD3AImyTzDnGmA0iMl1ExgKIyAgRyQduAN4UkQ3BDLpOHTtC375az6+UUjXw6wEuY8wCYEG1ZU9UmV+JrQKq7RjvAu/WO8KGGjkSvvqqyd5OKRVeauu7Py8vj2uuueb7jtuam9DqsqGqjAwoKIDdu52ORCmlmpXQ67KhUmVPncuXw3XXORuLUqpetj6wlaNZgR0Jq/XQ1vR7rd9Z1z/88MP07NmTu+++G4CnnnoKEWHJkiUcOHCA8vJynn32WcaNG3fWY9TkxIkT/PznPyczM5OIiAheeeUVLr30UjZs2MDkyZMpKyvD6/Xy0Ucf0a1bN2688Uby8/PxeDw8/vjj3BSEDidDN/EPGWL76lmxQhO/UqpOEyZM4IEHHvg+8c+ZM4eFCxfy4IMP0rZtW4qKivjBD37A2LFj6zXg+YwZMwBYt24dmzZt4qqrrmLLli288cYb3H///UycOJGysjI8Hg8LFiygW7duzJ8/H4BDhw4F/oMSyom/VSs78IK27FGqxamtZB4sw4YNY//+/RQUFFBYWEiHDh3o2rUrDz74IEuWLMHlcrF792727dtHYmKi38f95ptvuPfeewEYMGAAPXv2ZMuWLZx//vn85je/IT8/n+uuu45+/fqRlpbGlClTePjhh7nmmmu4+OKLg/JZQ7eOH2w9f2YmeDxOR6KUagHGjx/P3LlzmT17NhMmTOCDDz6gsLCQVatWkZWVRZcuXThx4kS9jnm2jjBvueUW5s2bR0xMDKNHj2bx4sWcc845rFq1irS0NB555BGmT58eiI91htBO/CNHwtGjkJ3tdCRKqRZgwoQJfPjhh8ydO5fx48dz6NAhOnfuTGRkJF999RU7duyo9zFHjRrFBx98AMCWLVvYuXMn/fv3Jzc3l969e3PfffcxduxY1q5dS0FBAbGxsfzHf/wHU6ZMCVrf/qFb1QOn99SZmupsLEqpZm/w4MEcOXKEpKQkunbtysSJE/nxj39Meno6Q4cOZcCAAfU+5t13381dd91FWloaERERvPvuu0RHRzN79mz+/Oc/ExkZSWJiIk888QQrV65k6tSpuFwuIiMjef3114PwKf3sj78pNbo//qq8XoiPhxtvhDffDMwxlVJBof3x+y/o/fG3aC4XjBihT/AqpVQVoV3VA7ae//nnobQUYmOdjkYpFULWrVvHrbfeetqy6Oholjfz1oShn/gzMmyrntWr4aKLnI5GKVULY0y92sg7LS0tjaysrCZ9z0BUz4d2VQ9oT51KtRCtWrWiuLg4IIktVBljKC4uplWrVo06TuiX+Lt0gZ49tZ5fqWYuOTmZ/Px8Gj0KX4hr1aoVycm19olZp9BP/GDr+bXEr1SzFhkZSUpKitNhhIXQr+oBW92zYwfs2+d0JEop5bjwSPyVPXVqdY9SSoVJ4j/vPHC7NfErpRThkvhjYyEtTev5lVKKcEn8YOv5V6603TgopVQYC5/EP3IkHDwIW7c6HYlSSjkqfBJ/1Z46lVIqjIVP4h84EFq31np+pVTY8yvxi8gYEdksIjkiMq2G9aNEZLWIVIjI+CrLh4rIUhHZICJrRSTwowb7y+2G9HQt8Sulwl6diV9E3MAM4GpgEHCziAyqttlOYBIwq9ryUuA2Y8xgYAzwmoi0b2zQDTZyJGRlQT2HTlNKqVDiT4k/A8gxxuQaY8qAD4FxVTcwxuQZY9YC3mrLtxhjtvrmC4D9QEJAIm+IjAwoL4c1axwLQSmlnOZP4k8CdlV5ne9bVi8ikgFEAdtqWHeniGSKSGZQO2iqfIJX6/mVUmHMn8RfU+fY9eo3VUS6An8CJhtjzmhIb4x5yxiTboxJT0gI4g+CpCTo1k3r+ZVSYc2fxJ8PdK/yOhko8PcNRKQtMB94zBizrH7hBYH21KmUCnP+JP6VQD8RSRGRKGACMM+fg/u2/xh43xjz14aHGUAZGZCTAyUlTkeilFKOqDPxG2MqgHuARUA2MMcYs0FEpovIWAARGSEi+cANwJsissG3+43AKGCSiGT5pqFB+ST+0p46lVJhzq+BWIwxC4AF1ZY9UWV+JbYKqPp+fwb+3MgYA2v4cBCxiX/MGKejUUqpJhc+T+5WatsWBg3Sen6lVNgKv8QPtp5/xQrQQZ2VUmEoPBP/yJFQVAR5eU5HopRSTS48E39lT51a3aOUCkPhmfhTUyEmRlv2KKXCUngm/shIOw6vlviVUmEoPBM/2Hr+1attp21KKRVGwjfxZ2TY7pnXrXM6EqWUalLhm/j1CV6lVJgK38TfsyckJGg9v1Iq7IRv4hexpX4t8Sulwkz4Jn6w9fzZ2XD4sNORKKVUkwnvxD9ypO22ITPT6UiUUqrJhHfiHzHC/tV6fqVUGAnvxN+hAwweDLNng8fjdDRKKdUkwjvxAzz2GKxZAzNnOh2JUko1CU38N90EF14Ijz6qN3mVUmFBE78IvPYa7N8Pzz7rdDRKKRV0mvgB0tNh0iR7AcjJcToapZQKKk38lZ57DqKjYcoUpyNRSqmg0sRfqWtX+PWv4e9/hy+/dDoapZQKGk38VT34IKSkwAMPQEWF09EopVRQ+JX4RWSMiGwWkRwRmVbD+lEislpEKkRkfLV1t4vIVt90e6ACD4pWreCll2D9evjDH5yORimlgqLOxC8ibmAGcDUwCLhZRAZV22wnMAmYVW3fjsCTwEggA3hSRDo0Puwguu46uOQSePxxOHDA6WiUUirg/CnxZwA5xphcY0wZ8CEwruoGxpg8Y8xawFtt39HA58aYEmPMAeBzYEwA4g6eyuadJSUwfbrT0SilVMD5k/iTgF1VXuf7lvnDr31F5E4RyRSRzMLCQj8PHURDh8JPfgL/8z+waZPT0SilVED5k/ilhmXGz+P7ta8x5i1jTLoxJj0hIcHPQwfZs89CbCz88pdOR6KUUgHlT+LPB7pXeZ0MFPh5/Mbs66zOnW09/4IFsHCh09EopVTA+JP4VwL9RCRFRKKACcA8P4+/CLhKRDr4bupe5VvWMtx3H/TrZ5t5lpc7HY1SSgVEnYnfGFMB3INN2NnAHGPMBhGZLiJjAURkhIjkAzcAb4rIBt++JcAz2IvHSmC6b1nLEBUFL79s6/lff93paJRSKiDEGH+r65tGenq6yWxOI2IZA6NH21G6tm6F+HinI1JKqTOIyCpjTLo/2+qTu3URgVdftV02P/mk09EopVSjaeL3x+DBcNdd8MYbsGGD09EopVSjaOL319NPQ9u29kZvM6seU0qp+tDE76/4eHjqKfj8c/jkE6ejUUqpBtPEXx8//zkMHGgf6iorczoapZRqEE389REZCa+8Ylv3/Pd/Ox2NUko1iCb++hozBv7t3+Cxx2y1j1JKtTCa+Bvivfegf38YOxa++MLpaJRSql408TdEfLxN+OecAz/+sQ7VqJRqUTTxN1SnTjbh9+tnk/9XXzkdkVJK+UUTf2NUJv8+fWy9/z/+4XRESilVJ038jZWQYJN/SopN/kuWOB2RUkrVShN/IHTuDIsXQ8+e8KMfwddfOx2RUkqdlSb+QOnSxSb/7t3h6qvhm2+cjkgppWqkiT+QEhNt8k9Otsn/X/9yOiKllDqDJv5A69rVtvDp1s0+7PXtt05HpJRSp9HEHwyVyb9rV5v8ly1zOiKllPqeJv5g6dbNJv8uXewIXsuXOx2RUkoBmviDKynJJv+EBLjqKlixwumIlFJKE3/QJSfb5N+pk03+q1c7HZFSKsxp4m8K3bvb5N+unU3+69c7HZFSKoz5lfhFZIyIbBaRHBGZVsP6aBGZ7Vu/XER6+ZZHish7IrJORLJF5JHAht+C9Ohhm3pGRcEVV8CWLU5HpJQKU3UmfhFxAzOAq4FBwM0iMqjaZncAB4wxfYFXgd/6lt8ARBtj0oDhwM8qLwphqU8f272D1wuXXw55eU5HpJQKQ/6U+DOAHGNMrjGmDPgQGFdtm3HAe775ucDlIiKAAeJEJAKIAcqAwwGJvKUaONAO4HL0KFx2Geze7XRESqkw40/iTwJ2VXmd71tW4zbGmArgEBCPvQgcA/YAO4HfGWNKGhlzyzdkCCxaBEVFtuS/b5/TESmlwog/iV9qWGb83CYD8ADdgBTglyLS+4w3ELlTRDJFJLOwsNCPkEJARgbMnw87d8KVV0KJXg+VUk3Dn8SfD3Sv8joZKDjbNr5qnXZACXALsNAYU26M2Q/8C0iv/gbGmLeMMenGmPSEhIT6f4qW6uKLYd48e6N39Gg4dMjpiJRSYcCfxL8S6CciKSISBUwA5lXbZh5wu29+PLDYGGOw1TuXiRUH/ADYFJjQQ8QVV8DcuZCVZfvzP3bM6YiUUiGuzsTvq7O/B1gEZANzjDEbRGS6iIz1bfZHIF5EcoCHgMomnzOA1sB67AVkpjFmbYA/Q8t3zTUwaxYsXWoHcD9+3OmIlFIhTGzBvPlIT083mZmZTofhjPffh9tvt4O5fPyxbfOvlFJ+EJFVxpgzqtJrok/uNie33QZvvAELFsAtt0BFhdMRKaVCkCb+5uZnP4NXXoGPPoJJk8DjcToipVSIiXA6AFWDBx+E0lJ47DGIiYG33gKpqcWsUkrVnyb+5urRR23yf+452L4d3nzTdvmglFKNpFU9zdmzz8Lrr9t+/NPS4He/03p/pVSjaeJvzkTgrrtg40b7dO/UqfCDH9g2/0op1UCa+FuC5GT43/+FOXNg1y5IT4dHHtH2/kqpBtHE31KIwA03QHa2bfb5wgu2s7d//tPpyJRSLYwm/pamY0d45x3btbPHAz/8oW0CevCg05EppVoITfwt1RVXwLp1MGUKvP02DBpkn/ZVSqk6aOJvyWJj4aWXbKufzp3huutg/HjYs8fpyJRSzZgm/lAwfDisXAnPPw+ffGJH+dLSv1LqLDTxh4rISJg2Ddauhf797Y3gv//d6aiUUs2QJv5Qc8459sZverpN/p9+6nRESqlmRhN/KGrb1ib81FRb7//ll05HpJRqRjTxh6oOHeCzz6BvXzu4y9dfOx2RUqqZ0MQfyjp1gi++gO7d7bCOy5c7HZFSqhnQxB/qunSxVT0JCTBmDHz3ndMRKaUcpok/HCQlweLFtu7/yith/XqnI1JKOUgTf7jo2dOW/KOj4fLLYdMmpyNSSjlEE3846dv3VAufyy+HbducjUcp5QhN/OFmwAB7w/fECbjsMtixw+mIlFJNzK/ELyJjRGSziOSIyLQa1keLyGzf+uUi0qvKunNFZKmIbBCRdSLSKnDhqwZJS7MPeR06ZEv+u3c7HZFSqgnVmfhFxA3MAK4GBgE3i8igapvdARwwxvQFXgV+69s3AvgzcJcxZjDwQ6A8YNGrhjvvPFi4EPbtsz197tvndERKqSbiT4k/A8gxxuQaY8qAD4Fx1bYZB7znm58LXC4iAlwFrDXGrAEwxhQbYzyBCV012g9+APPn2+qeK66AoiKnI1JKNQF/En8SsKvK63zfshq3McZUAIeAeOAcwIjIIhFZLSK/anzIKqBGjYJ582DrVvsr4PnntfSvVIjzJ/FLDcuMn9tEABcBE31/rxWRy894A5E7RSRTRDILCwv9CEkF1BVXnOre4de/tk/63nQTfPUVmOr/1Eqpls6fxJ8PdK/yOhkoONs2vnr9dkCJb/k/jTFFxphSYAFwXvU3MMa8ZYxJN8akJyQk1P9TqMYbNco+5JWdDffcY2/+XnaZ7dv/1VehpMTpCJVSAeJP4l8J9BORFBGJAiYA86ptMw+43Tc/HlhsjDHAIuBcEYn1XRAuATYGJnQVFAMGwCuv2JY+771nx/h96CH79O+kSbBsmf4KUKqFqzPx++rs78Em8WxgjjFmg4hMF5Gxvs3+CMSLSA7wEDDNt+8B4BXsxSMLWG2MmR/4j6ECLiYGbrsNvv0WsrJg8mT46CM4/3wYNgzeeAOOHHE6SqVUA4hpZqW39PR0k5mZ6XQYqiZHjsCsWfD667BmDbRube8FXHMNXHoptGvndIRKhS0RWWWMSfdnW31yV/mvTRv42c9sD5/LllEx9mYOfLABrr0W4uPhootg+nRbHVRR4XS0SqmziHA6ANUCicDIkeR1iSf/RD5DXhM6FH5uWwY99RQ8+SS0b2+fCr7qKjv16uV01EopH63qUQ3iLfOyNGkp5UXlxKXFMXz1cFwRLiguth3BffYZLFoE+fl2h379Tl0ELr3U/npQSgWMVvWooCueX0x5UTmJ/5nIsXXH2PPmHrsiPh5uvBHefht27rTNQ3//e5v4Z86EcePs4DDffuvsB1AqjIVU4vcc194gmsremXuJ6hrFOW+eQ/vL2rP98e2UF1frhknENg+97z7bNURJiX1WID4eHnxQm4Uq5ZCQSfzHc4+zctBK9v91v9OhhLyTe09SvKCYLrd2wRXhou/v+1JxuILtj2+vfcfoaFvNM306rFgBc+c2TcBKqdOETOKPToomKjGKTZM3cWzDMafD+Z7xGg58dYBNd2xi9fmrObnnpNMhNdq+P+0DDyROTgSgdWprkn6RRMGbBRzJ8qNt/223QWqq7R6iXDtrVaqphUzid0W7GDx3MO7WbtZfu56KQ842Jzy6/ijbpm1jWc9lrLlsDYVzCjny3RG23LmF5nZDvT6MMeyduZe257clbkDc98t7PdWLiA4R5NyXU/fnc7vhhRcgJwfeeivIESulqguZxA+21D/4r4M5sf0E2bdlY7xNm2BP7jnJrpd3kTksk8y0THb9bhdxQ+IY+JeBXLDvAnq/0JviT4rZO3Nvk8YVSEdWHKE0u/T70n6lyA6R9H6uN4e+PsT+2X5Ut/3oR3DJJfD00/oEsFJNLKQSP0D7i9vT5+U+FM8rZsdzwR9WsOJoBXvf38uaq9awNHkp26ZsQyKFvv/VlwsKLuDcT86ly4QuuGPdJN+XTLtL2pHzQA4ndpwIemzBsGfmHlwxLjrf1PmMdV3v6ErrYa3ZNmUbnmN13GgXgRdfhMJC+N3vghStUqomIZf4AZLuTaLzxM7kPZFH8afFAT++t8JL8cJiNk7cyLddvmXT7Zs4nnOcno/2JGNTBsNXDCf53mSiOkedtp+4hAEzB4CBTZM3NfkvksbyHPew/8P9JFyfQETbM5/9E7fQ77/7Uba7jJ0v7Kz7gBkZcMMN8PLLsLfl/gpSqqUJycQvIvR/qz9x58aRfUs2x7cdD9ixy/aVkXVJFuuuXkfJpyV0ubULw74ZxshtI0mZnkJs/9ha949JiaHPq304+NVBds9oWWPdFn1chOeQ54xqnqraXdiOzhM7s/OlnRzP9eO8P/ccnDxpq3yUUk0iJBM/gDvWTerfUkFg/bXr66568MORrCOsGrGKo98dpf87/blgzwX0f6M/7S5shx1p0j9d7+hKx6s7kvtwLqVbShsdV1PZ884eWvVqRfsftq91uz6/7YNECNt+ua3ug/bta/v/+cMfYPPmAEWqlKpNyCZ+gJjeMQycNZBj64+x+c7NjWpNU/i3Qr678DswMOxfw+g6uSuu6IadPhGh/9v9cbVysen2TXgrvA2Oq6mc2HGCg4sPkjgpEXHVfpGLToqm56M9KfrfIko+92MAlyeesN1AP/pogKJVStUmpBM/QPyYeFKeSWH/rP3s/q/6V60YY9jxmx1suH4DcWlxnLfiPNoMa3w/M9Hdouk3ox+Hlx1m10u76t7BYXvf2wsGutzexa/tkx9MplWfVuTcn4O3vI4LW+fOMHWq7e9/2bIARKuUqk3IJ36AHo/0IH5cPDm/zOHgPw/6vZ/nuIfsidlsf2w7nSd2Zug/hhLdNTpgcXWe0JmEGxLIezKPo2uPBuy4gWa8hr3v7qX9Ze2J6RXj1z7uVm76vtqX0uxS/+5lPPSQ7cPnV7/SrhyUCrKwSPziEga+N5CYPjFsuHEDJ/Lrbkp5suAkWZdksf/D/aQ8n8LAPw3E3cod2LhE6Pf/+hHRMYLs27LxljXPKp+DSw5yYvuJWm/q1iT+mng6julI3pN5lO0vq33j1q1tl85ffw2ffNLwYJVSdQqLxA8Q0S6C1I9T8ZZ62TB+A96TZ0+yhzMPs2rEKo5tPEbqx6n0nNazXjdv6yOqUxT93+rPsTXHyJueF5T3aKy9M/fibusm4bqEeu0nIvR9rS/eUi+5v86te4c77oBzzoFp03QgF6WCKGwSP0DcoDgGvDuAI8uPsPX+rTVus3/OfrJGZSGRwnnfnkencZ2CHlensZ1InJTIzud3cnj54aC/X31UHK6g8K+FdJ7QGXds/X/xxPaPJen+JPa+s5fDmXV8tshIeP552LjRDvSulAqKsEr8AAnXJ9D94e7seXMPe/645/vlxmvY/uR2Nt60kTbD2zB8xXBan9u6yeLq+1pfopOiyb4tG09p8+leev+c/XiPe+tdzVNVryd6Edk50vbjU9dDa9deawd0f+IJKG05TV2VaknCLvEDpDybQocrOrDl7i0cXnEYzzEPG2/ayI7pO0icnMiQL4ac8dRtsEW0i6D/O/05vuU42x+to3vjJrR35l5iB8TSdmTbBh8jom0EvV/ozeGlh9n3wb7aN67syqGgwA7gopQKuLBM/K4IFwP/MpCorlFsuH4D3138HYV/K6TPy33o/8f+DW6f31gdr+hIt190I/+1fA7844AjMVRVurmUw98eJnFyYqPvcSTelkibjDbk/iqXiiN11N9fdBGMHWt78CwqatT7KqXO5FeGE5ExIrJZRHJEZFoN66NFZLZv/XIR6VVtfQ8ROSoiUwITduNFdYoi9W+plBeVc3zbcdL+L43uD3UP2k1cf/X5bR9i+sawadKmuhNkkO19dy+4ocut/rXdr424fP347C1jx7N+dJ73/PNw9Cj85jeNfm+l1OnqTPwi4gZmAFcDg4CbRWRQtc3uAA4YY/oCrwK/rbb+VeDTxocbWG3Oa8Owb4eRnpVO/I/inQ4HAHecmwHvDeDkrpNse8iPLg+CxFvhZe97e4m/Oj5gzy60zWhL4uRE8l/NpzSnjvr7QYPgP/8TZsyA7c2n6kupUOBPiT8DyDHG5BpjyoAPgXHVthkHVDbDmAtcLr6is4j8O5ALbAhMyIHVZlgbYlL8eyipqbS7oB3dp3Znz9t7KF4Q+N5F/XHgswOU7Slr1E3dmqQ8l4JECDue9qPU//TTEBEBjz0W0BiUCnf+JP4koGqfAvm+ZTVuY4ypAA4B8SISBzwMaNeL9ZTydApxqXFs/slmyorqePgpCPbO3Etkp0jirwnsL6HoxGiS7kli36x9HNtUxxCZ3brZQdlnzYLVqwPNFuyYAAAQVElEQVQah1LhzJ/EX1Old/U2eWfb5mngVWNMrf0RiMidIpIpIpmFhYV+hBT6XNEuBrw/gPLictb9aB0Vh5uuvr+8uJyieUV0ntgZV1Tgb3R3n9odV4yLHdP9KPX/6lcQHw8PPxzwOJQKV/78r84Huld5nQwUnG0bEYkA2gElwEjgRRHJAx4Afi0i91R/A2PMW8aYdGNMekJC/Z4ODWVthrVh8F8Hc/S7o6y7Zl1Aupb2x75Z+zBlhq6Tuwbl+FEJUSTfm8z+D/dzbEMdpf527eDxx+GLL7QrB6UCxJ/EvxLoJyIpIhIFTADmVdtmHnC7b348sNhYFxtjehljegGvAc8ZY/4nQLGHhU5jOzHwzwM59K9DdlyBE8FP/ntn7qX1ea1pPSR4D7B1n9Idd5zbv24q7rrL3uy95RZYuTJoMSkVLupM/L46+3uARUA2MMcYs0FEpovIWN9mf8TW6ecADwFnNPlUDdf5ps70/2N/Dnx+gI03bqy7m+NGOJJ1hKPfHQ34Td3qIuMjSbo/icI5hRxdV0fPpNHR8Nln0KkTjB4Na9cGNTalQp00ZnCSYEhPTzeZmZlOh9Es7f5/u9n6i60k3JTAoA8GIe7AP3Ow9f6tFLxRwAV7LiCyY2TAj19VeUk5y1KW0eHKDqTOTa17h+3b4eKLobzc9uJ5zjlBjU+plkREVhlj0v3ZNiyf3G2pku5OoveLvSmcXcjmn2wO+GDt3jIv+z7YR6dxnYKe9AEiO0aS/EAyRR8VcSTrSN07pKTYun5j4PLLIS8v6DEqFYo08bcwPab2oOeTPdn77l623re1UcNJVlf8f8VUFFcEvZqnquQHk3G3c5P3VJ5/OwwYAJ9/DseO2eRfUL2dgVKqLpr4W6BeT/ai+5TuFMwoIHdabsCS/56Ze4hKiqLjVR0Dcjx/RLaPpPsvu1P892KOrPKj1A8wZAgsXAj798MVV4A2AVaqXjTxt0AiQu8Xe9Pt7m7senEXO57xoz18HU4WnKTk0xISb0sMyr2D2iTfn0xEhwj/S/0AGRkwf76t7rnqKjjo/5CaSoU7TfwtlIjt9CxxUiJ5T+ax6+WGD9h+cvdJ26zSC4mTmq6ap1JE2wi6T+lO8SfFHF5Rj4FoRo2Cjz+GDRvg6qttp25KqTpp4m/BxCX0f7s/CTcmsG3KNna/7seg5oDxGA4tO0TuY7lkDstkafJS9ry5h07XdyL2nNggR12zpHuTiIivZ6kfbPPO2bNt+/6xY+H48aDEp1QoiXA6ANU44hYG/nkg3uNett69FXesm8Tbzyy1lx8s58BnByj+pJiST0soLyoHF7S7sB29X+hN/DXxxA5yJukDRLSJoMfUHuROy+XQ0kO0O7+d/ztfe60dqvHWW2H8ePsrIKppB9JRqiXRxB8CXJEuBs0ZxPqx69n0n5twxbhIuCGB0k2lFM8vpmR+CQe/PggeiOgYQcerOxL/b/F0HN2xSZpt+qvbL7qx6+Vd5D2Vx5BFQ+q388SJdqjGO++083/5i+3ZUyl1Bv2fESLcrdykfpzK2jFryZ6YTe4juZzIPQFAXFocPab2IP6aeNqMbIMronnW8EW0jqD7r7qTOzWXg98cpP1F7et3gJ/+1DbzfPBBiI2FmTPB1Tw/q1JO0sQfQtxxbtLmp7Hxlo1gbH848f8WT6serZwOzW9Jdyex63e7yHsyj6FfDq3/AR54wN7kffxxiIuzA7k4PKqaUs2NJv4QE9E2gnM/OdfpMBrMHeumx7QebHtwGwf/eZD2l9Sz1A/w6KM2+f/2t5CbCxdeCGlpkJpqn/51uwMfuFItiCZ+1ex0+5l9PmH7E9sZ+o+h9R8HWcSO2RsdDX/6EyxadGpdTAwMHmwvAqmppy4IXbvqLwMVNrSTNtUs5f93Pjn35TDkyyF0uKxD4w529Chs3Ajr18O6dfbv+vWwd++pbTp2PHUxGDoUfvhD6NtXLwaqxahPJ22a+FWz5DnhYXnf5cSkxDB0SQNK/f4oLLQPf1W/IBz2PUSWnAyXXWanSy+FHj0CH4NSAaKJX4WEym6oz/3sXDpe2UT9BxkDW7bAV1/B4sX2b1GRXdenz6mLwKWXQmLTP+Ws1Nlo4lchwXvSy/J+y4lOimbYt8OCU+qvMwiv/RVQeSH45z/h0CG7btAgewG47DJbNdSx6Tq3U6o6TfwqZBS8VcCWn20h7dM04sfEOx0OeDzw3Xf2IrB4sR0QprTU3gsYMQLGjLHdSGRk6ANkqklp4lchw1vmZfk5y4nqHMV5y88LaKnfW+7Fc8yD95gXz1EPnmN28pZ68ZR68J7w4j1uJ89xz/fzp70+VoF3TwkUFdHuxAo67ppLnMlF2reHK6+0F4HRo+39AqWCqD6JX4skqllzRbno9XgvNv9kM+uvXY87zg0GMNhxCHzzNb72GkyZ+T6he4558Bz1JfpjHkx5/Qs9EiG4Yly4Wrns3xgX7pg4vK4YinfGk8vVRHfyEN8ph45f/h8d/noPbk7Y1kKVvwYuvtg2NVXKIVriV82et9zLumvWcXzbcVviF76fan2NvXC4W7txxblwx7lPTTUsc8XZbd2x7ipJ/dS8K8ZVa3cXJ/JPUPJpCSULSij5vATvMS8SBe17HSKeZcRv/wsx5TtsdxKXXmrHEejVyz5hXNMUG6tdTii/aVWPUg7znvRy8OuDlCwooXhBMcc32+6iY5K8xHfaRsfC/6N9wUJclNd+oJgYiIvDxMZh4tpAbGu8fQbiuehyvBmj8MR2sFVTxzy2euqYraY6rcrqmBfjMbhiT13oapyPrXIx9F38xC2nX1TryRgDXtsVeOWEB0yFOX2Zl1O/2qrOG+z+Nc1XTlRbRpX9OX07sD3aSoSdcNtfcd8vq/43QsAFpszYqr/K6aT39Nc1rBOXINGCK9qFK8p1at431bTOHesmol3DKmI08SvVzJTmlH7/a+DAVwcwJw2uGCEiDl8yNLZqygPGAxiD8QrGC5jGlforE7in1GOTamNU/XXlqnZRcPnWeTg9oSu/tclow/Dlwxu0b8Dr+EVkDPB7wA28bYx5odr6aOB9YDhQDNxkjMkTkSuBF4AooAyYaoxZ7PcnUSpExPaNJfbeWJLvTcZzzMOBrw5w4IsDeI56bOnSN+Hi1LzbN++qMu8WXFGCq7gAd846XBu/w70pC7e3FFcrcGcMwX3Z+bhGX4I7tR+uWLfdH1sC9p70nvqFUPmr4GzzpZ4aS9nGW/N85esz4q/+OqKGZZUXEdfp1XWnXVyqzNdUrXfaMjht3ffLTZVfHxXm1C+Pimq/RKr9dUX77utU/q1pqrJOogW89pefKfOdd99kTtbwuszOR8Y3TTfpdZb4RcQNbAGuBPKBlcDNxpiNVba5GzjXGHOXiEwArjXG3CQiw4B9xpgCEUkFFhljkmp7Py3xK1VPR47AP/5h+yRatAhycuzynj1PtSoaOhROnrRNT/2dKirsQ2pJSadPCQnO3Hs4eRK2b7efLyfHjrPcurW9H1L5t+p81b+xsSHfOV9Aq3pE5HzgKWPMaN/rRwCMMc9X2WaRb5ulIhIB7AUSTJWDi60gLAK6GWNOnu39NPEr1Ui5ufYC8Nln8OWX9sLgL5HTbyzv328fYqsqMtJ2alf9gpCUBN26Qbt2Z96ojvSzJFtaauOvTO5Vp5077ZPVDdWqFbRvb5/A7tvXTv36nZpvV49R3wLJGDhwAHbvts+JDG1Ad+QEvqonCag6knc+MPJs2xhjKkTkEBCPTfSVrge+qy3pK6UCoHdv+PnP7VReDsuW2cQZG1v3FBV1esd0Hg/s2wf5+TYxVZ/WrYOFC+se6D4q6uytl+LibLcYOTn2mFXFx9ukfNFFpxJ05dShg71QHD1qB+A5dqzu+eJi2LYNvvjCDtdZVadOp18IKi8MvXvbcxMRYaf63OQ+eRL27Kn53BUUnJo/YQdNYuRI++8VZP4k/po+ZfXLbq3biMhg4LfAVTW+gcidwJ0APbQjLKUCJzLSPjdw8cUN29/ttqX4bt1q3+7w4VPJ7MiRMxNv1anq8qIi2LHDdndxxRWnJ90+fWxyr02bNnZqiNJSexGo+qti61ZbbfanP519P5fLXgAiI09dDKpPbjeUlJzq56mq6OhTv5BGjIB///dTr1NSGvZZ6smfxJ8PdK/yOhkoOMs2+b6qnnZACYCIJAMfA7cZY7bV9AbGmLeAt8BW9dTnAyilmoG2be00cKDTkfgvNtaOx5CWdua648dPVTnl5dkSeUXFqam8/PTX1ZeVl9uLVvVqsKQke5FzuLtvfxL/SqCfiKQAu4EJwC3VtpkH3A4sBcYDi40xRkTaA/OBR4wx/wpc2EopFUSVA/YMHux0JEFR5615Y0wFcA+wCMgG5hhjNojIdBEZ69vsj0C8iOQADwHTfMvvAfoCj4tIlm/qHPBPoZRSym/6AJdSSoWA+rTq0Y5AlFIqzGjiV0qpMKOJXymlwowmfqWUCjOa+JVSKsxo4ldKqTDT7JpzikghsAPoxOl9/YQzPReWngdLz4Ol58GqPA89jTEJ/uzQ7BJ/JRHJ9LdNaqjTc2HpebD0PFh6HqyGnAet6lFKqTCjiV8ppcJMc078bzkdQDOi58LS82DpebD0PFj1Pg/Nto5fKaVUcDTnEr9SSqkgaJaJX0TGiMhmEckRkWl17xGaRCRPRNb5urMOqy5LReQdEdkvIuurLOsoIp+LyFbf3zqGZ2r5znIenhKR3VW6Ov+RkzE2BRHpLiJfiUi2iGwQkft9y8PqO1HLeajXd6LZVfWIiBvYAlyJHdlrJXCzMWajo4E5QETygHRjTNi1VRaRUcBR4H1jTKpv2YtAiTHmBV+BoIMx5mEn4wy2s5yHp4CjxpjfORlbUxKRrkBXY8xqEWkDrAL+HZhEGH0najkPN1KP70RzLPFnADnGmFxjTBnwITDO4ZhUEzPGLME3fGcV44DKEbLfw37hQ9pZzkPYMcbsMcas9s0fwQ4KlUSYfSdqOQ/10hwTfxKwq8rrfBrwwUKEAT4TkVW+AenDXRdjzB6w/wGAcB7N7R4RWeurCgrp6o3qRKQXMAxYThh/J6qdB6jHd6I5Jv6aRiFuXvVRTedCY8x5wNXAL3w/+5V6HegDDAX2AC87G07TEZHWwEfAA8aYw07H45QazkO9vhPNMfHnA92rvE4GChyKxVHGmALf3/3Ax9hqsHC2z1fHWVnXud/heBxhjNlnjPEYY7zAHwiT74WIRGKT3QfGmL/5Fofdd6Km81Df70RzTPwrgX4ikiIiUcAEYJ7DMTU5EYnz3bxBROKAq4D1te8V8uYBt/vmbwf+7mAsjqlMdD7XEgbfCxER4I9AtjHmlSqrwuo7cbbzUN/vRLNr1QPga4r0GuAG3jHG/MbhkJqciPTGlvIBIoBZ4XQeROQvwA+xPQ/uA54E/heYA/QAdgI3GGNC+sbnWc7DD7E/6Q2QB/yssp47VInIRcDXwDrA61v8a2z9dth8J2o5DzdTj+9Es0z8Simlgqc5VvUopZQKIk38SikVZjTxK6VUmNHEr5RSYUYTv1JKhRlN/EopFWY08SulVJjRxK+UUmHm/wOa333lPmySjgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8a6f7a0be0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 0.02674\n",
    "plt.plot(hist.history['acc'], 'b-', label='acc' )\n",
    "plt.plot(hist.history['val_acc'], 'g-', label='val_acc' )\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "plt.plot(hist.history['loss'], 'r-', label='loss')\n",
    "plt.plot(hist.history['val_loss'], 'm-', label='val_loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('cnn_final_checkpoint.hdf5')\n",
    "\n",
    "pred = model.predict(test)\n",
    "final = pred.argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({'ImageId': test_index+1, 'Label': final})\n",
    "submission.to_csv('cnn_submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py36",
   "language": "python",
   "name": "py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}