Skip to content

Instantly share code, notes, and snippets.

@artificialsoph

artificialsoph/CIFAR100.ipynb

Created November 28, 2017 20:38
Show Gist options
  • Save artificialsoph/99891233a22eef1f46827be533b46ca6 to your computer and use it in GitHub Desktop.
Save artificialsoph/99891233a22eef1f46827be533b46ca6 to your computer and use it in GitHub Desktop.
Download ZIP
CIFAR image recognition demo
Display the source blob
Display the rendered blob
Raw
CIFAR100.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-21T19:50:52.997278Z",
"start_time": "2017-11-21T19:50:52.760678Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-21T22:18:52.406508Z",
"start_time": "2017-11-21T22:18:52.403727Z"
}
},
"outputs": [],
"source": [
"import keras\n",
"import keras.backend as K"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-21T19:50:54.089966Z",
"start_time": "2017-11-21T19:50:54.087327Z"
}
},
"outputs": [],
"source": [
"from keras.datasets import cifar100"
]
},
{
"cell_type": "code",
"execution_count": 150,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-22T15:57:01.728078Z",
"start_time": "2017-11-22T15:57:01.568961Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(50000, 32, 32, 3) (10000, 32, 32, 3)\n"
]
}
],
"source": [
"(x_train_orig, y_train), (x_test_orig, y_test) = cifar100.load_data(label_mode='fine')\n",
"num_class = len(np.unique(y_train))\n",
"print(x_train_orig.shape, x_test_orig.shape)"
]
},
{
"cell_type": "code",
"execution_count": 152,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-22T15:57:18.689981Z",
"start_time": "2017-11-22T15:57:18.610862Z"
}
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAJIAAACPCAYAAAARM4LLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEatJREFUeJztndlzXNdxxu/c2TcMZrAQAEESImnu\nksjIUhKpqNBLbJfjF7tSqXKl/JIqV+VfyYtfXH52KlV5STmbEzkVxQ4dmYxNUbK4CyIJEMS+cIDZ\nt7v57X79XZcssHwysJX+PTXYhzMXg57Tffr06RMLgsBSlN8W+6AfQPl0oIakGEENSTGCGpJiBDUk\nxQhqSIoR1JAUI6ghKUZQQ1KMkBjy+/3/SKO7vVDcWF4g1Y13fxnKl7/4FdKNjU/89m8t5I6Ln5qt\nXRq3uPBhKJfH8qRbXn4Uyl9985ux/byvzkiKEdSQFCMM27WF+L5/UG9tHN9z6eeYsxfKze1F0l39\n4T9B1+yR7lvf/rZ4UXw+nh+JCMTXP7DY8zhi7PrGcijv1lZp3MbK/VBefPSMdPUGnt9685vWftAZ\nSTGCGpJiBDUkxQgHFiNFse3fL5uWEZ7te6z0mqEYdHdIlfcHoVzd2CTdpvg5HsPnURot0bhUKom3\niswFQYB4LYFhluN1adzYobFQ3trhGGljYd16Xn6//nrK7yxqSIoRfmdcmwmkgwk8Xlq7e5i+u/UW\nxqU4qztyeAY/2HHSxYQLi/twIfWNFRq3dO8Xofzkw3nS2XYqlBtieW5ZlvXTH/1jKJdnjoTy629c\npnFWYiQUq7U9UvVbcI+93nYoB26Txm3vIi2xV2PXFvjPP7/ojKQYQQ1JMYIakmKET1WMZImtimeP\nOTbZfv9aKHd266G8OeDv0qnLV0L5My9/lnS2WHbfuftBKH9w9SqNa4qYqbG9RbpkIh3KvSovs6++\n9TSUz/7Jl0P5j9/8Ao3r9ZFC2Nt+SrrFmz8K5a11VB6MHTtK4zp+O5SdDptByp60nhedkRQjqCEp\nRvhUubag1w/l6kdcUGbVGqFYiYvdentAwxbf+a9QTgS8s56ZgXv4ux/8Wyjff+8WjTteRkqhYnNl\nQD6Jj9yLJ0m3+BCu7trDH4Ty9Ox5Gnf5tbOhvDP/v6S7/fY/h3JfpAbaa+doXO7cK5Cz46QrvlC2\nnhedkRQjqCEpRjg41/Z/YMJ2ClnjwuQM6XZWn4RybwdFXvkUF9g1enBn87+4RrpO+Vgov/32dfx7\nk7PGRXsacjlDunYfrm5+ObJp20ZR2moVbunv//b7NG71FlZVnZX3SJf3sBpLZ7FC7Lc7NO5YAe7M\nPnSSdL1YynpedEZSjKCGpBhBDUkxwsHFSNHa/99k0lRF9vHDgiSW01Mvvkw6p1UL5YXlj0K5s8uF\nZ4N0NpQfPvyQdO0CisMSDuKZRpXPjNXFObHMsWnSNfYQ+9x5yjHSzgCxSbGEYrblx7dp3I1dVDZ8\nZpxTCKkknqvWh1yczNK4jXVk30dyFX6Nypj1vOiMpBhBDUkxwoG5NjtyVOs3nXKzA6ElN8ffg5gH\nZTLNy+7Dr72BH4Q32PjldRo3KwrKqs+4FvvODWzUZhNwc+NFXi5fuYz3+sOXOaP83e99L5SbXc6q\ny2eWhWidyNI9fQSuxw+4gG9rGxn8RPlQKMfyfBz89n1k/uvv8wb39PHjofznn/+StR90RlKMoIak\nGEENSTHCUGMkT8Qw8YhO9gLoDfqkSyXwmPK8V/Q1LKFzI1HXwi4K3PdELNI/dYHGnX/l9VB2lnlZ\n/w9v/Ri6LrYivv6VKzTuG19DXPHoMZ/9324j7hoE/BskA+hSCeiKGY738qOId+pOm3WHkG4Isjgk\nsLrD2zheFzHeQFRGWJZlXf3hPfzwN9+x9oPOSIoR1JAUIwzVtfUdLHczKV4yNzo4a3b95g3SjRQK\noXzp/Ev49xyfSfM8J5TXdrge+qfX4JaeLOM8WT+yBE/PzIWyG2k7s/0U9dGtJp73xNwRGpcQJ+xq\ndXYbAx8uy/XY/foduB87QI4inuHPqrqL7PjWNp9Jy4pzevkSQoTCKH9WReE6swnOxRwZH7WeF52R\nFCOoISlGGKpri4nptNFqke7mLTTpXN5YI106hQKtiQoKsk7PnaBx9UY1lG/d4qK0jaUHoby5DHew\nvcfPcesuaqBfmz1DuuNTWC3tVbDRWRrnjdmVdWzGbmywi2034ZZGC7yR2m7BtTX2sGI8PjlL4woZ\n/Nk6Wf4TeqIBqdfGe3l2xMWWxcZsgjP4pRI/137QGUkxghqSYgQ1JMUIw81s9+GLr994l3Tv378T\nyifOcEywvoIj1v/y7z8J5a991aFxC0soRFtYeUI6O47s8K5YMq+tLtG4jPdqKL84N0e6v/6rb4Wy\nXNafiHRUW19HjPfo7gPSNasopCuNcQGZ5+IZ8yIzcLhcpHGBOIsXi3QHjouyingcBxlchz+rjij0\niyc4veD5fBZvP+iMpBhBDUkxwlBdW7MFF/Xf7/yYdGMzWNb3e5xRfrqI5XRMTN3v3uGitHvCPcYi\nv1pc/pxAxvfKFy7SuMkylvVuh7PeF06fDmVb1F6v/udPaFz2GdzGnxa5s8fUKWTm39vZIN18Ftns\nuVmkFCYy/Lv0ekgT/Fp23IcLi4tupOkEL+kHIoueyuZIZyfT1vOiM5JiBDUkxQhqSIoRhhojJfPw\nvaVKgXRrayhGv3P7HumePsY2xvQsfP3YFKf9fbFs3dvlrY+kiK3mjiNumZrhpXW3jxhj0OMYyROV\nAt0lLPE7Sxzr1OuIn7KR1MCrR5HamE7ze4+IDm4J0RrHT/JyPPAQ+8R8XtZ7DuLLmAx1/I/v0Ov2\nOSZN2b9WMviJ6IykGEENSTHCUF3bjQ+QefYi9crxOB7lySJnpdfW4KYKZezAex53Fms2cf4r6tpe\nEC5lcgKubXX1IY0rJ7B0T57nJXOijjrnlVu47+x+g+um33oAXd1ntzGawVL7S6e52enrKRTIrWwt\nhXK8xMey3Rwy1k7ELQXirpPAx2cadV+yCDAeRFIIiec3C52RFCOoISlGGKpre7J0F28cqROeHENm\nOxY5SpTJwg1+8fPoP33m3HEa5/VRHDdZidQhT6OR6EQFq6XjR07TuKMT6PQWj3zN6uuo2a42cM/H\nosUrp+JLyF67XV5Z1kSP7399yhu65yeRzX5BLrk2+YqsbgkrrsDlo1uuC9fmO/I6Lv48Oj24/kye\nC9tSWc1sKweEGpJiBDUkxQhDjZFm5uDry+O84+yIM29f/rNXSVetik5pGXHkecCZ50uX0Ni81+bY\nYV0U/F88i3En5o7RuNozxDQbm1y4v7uCbrj2Sfy/y5+7QuN6NmKTRovjG1eEI/c/uku65Y8eh/Jk\nHDHNiM0xYyCuZLdjrIuJ7H4g3syNtBEaiEK3hMeN6V2Xn3k/6IykGEENSTHCUF3bOzf/I5Rdl5ec\nR+eQsb74Onc5e7qAwjY7Bvey26rSON9DmqBZ543OagMu693bWILPL/DG6doaxmUi2eAzadRY23mk\nCTbr7Aqu3/xZKLuRVnRJ0ey03oo0Qk3i+esZuMdEnHcBOhaey4vc8B0XWemEkB2XPw9bdnVJ8Ov3\n+hwW7AedkRQjqCEpRlBDUoww1BjpxEnEGI7LS/fJKblk5us1m22cg0+IqzwdjzuZ1ZuIb5zIercy\nixgsmUaMFM/wzv2xM/hu+R5/z4oJxFM/u4ZKhvuPuFdBsYi2MDE7UrgvutFVa9wRzg8wNhCHEJp7\nfCV7d4Aqh1iMl+4p0S5Iyt3IgYqEuFbVjnQHdiNx137QGUkxghqSYoShurbPXsROeyuS8X3wAPdt\n7NZ4Kj9zDg1Di4URoeFpfXsH7swZsK5ZEy1j2lh2j1WmaNxYBcVyrR5/zzJxuKxEDm7Oc/h3ScVQ\nj54rcKc0W7jH2s4K6Uan50K5nMKfpr7LxXd+DGFBOs3HrW3h6lwX2Wu5c2BZlpUXZ9m8SI4iX+A6\n8/2gM5JiBDUkxQhqSIoRhhoj1VvYgbctrsJr1OGn5+d56+Dx4v+E8uxRVFK+dJFb/x0Vuqw9QrpA\n7HB7YnsmleQC/5ios891OYUwncP7XbqIGGO8xPedXX8HPQnqezXSya2hnbVt0gV5pEe8U+J3i+zO\nywqIdIIPBnTFBTi+h22RVIbnjLioQh10I8t9zqrsC52RFCOoISlGGKpry6Vgt0Gk09gbf/RKKJ84\ncZZ0i0+XQnlbXLVeq/LZtYxox7LVZfc4OgpXVyxiCR4kI2mCBrLelTx3jpuYRHa8eQQu8ebPf07j\nqjW4cN//+JvoYhEXUqngHyqHkWpoR77uSbFzn8pGjlfH4I674r6RIHJBnisK4KKP2OlqYZtyQKgh\nKUYYqmuz45hO7SRPtSPiWPL41GHSnb2AIrJeD9OuH9lc3HiGriDbdb6jY7uxFcpT03BRpRL7F9+G\nu2w5/D2r9tBAdW0XG8T3HnDnuH4P753JfPwSKF+KnL2riGx2E/el2KP8GqNJrE59izPWcgPWFdd2\nybtTLMuy4rLjSOQAX2SRuC90RlKMoIakGEENSTHCUGOkh+s4t1Ua5aL79AAxx0iGd8zLYrmeERla\n2+Kd70lxUUsy0sW10UQ6IB4gCGjUOPO8tYMDBfUtLrB7PI4KhdnSpVD+y794k8bdvYlx0bN3o2VU\nF/QjWfWghtTDvQfo0Ds3wd3txvKi826bi+OqIps9kkQKIYgUwLXqqIbIRO69y43w++0HnZEUI6gh\nKUYYqmurteC+ei7XEKdFHbVT5MKqJt3thjRsLstTciGHtjCZFE/PEyVkth1RiCbrvC3LslYfi4ag\nkXrrO1soRFsRK/JTKc7EV8Tzz0zOkM4WGeVejt1NNYlN3MMW3Hk2wZ9HNi+K6jqcGnA82UwV9eHO\ngM+1dURhYTrSFLVc5mK//aAzkmIENSTFCGpIihGGGiPNHjoZym6k4NwWafpu5Ar17RrOnsll/JFj\n7Ms7ohC+1+TzagVx5fuYuCctmeT2OsePYTmdK3D8sbiAbYV0AvGZPc2/y+ghxGMtcU+tZVlW3EPc\ncuL8SdL589jScMTdbZk0P6Mn2tyMFViXEP0D9p4hlRHzuZCw0xVtbdKss+Pa1VY5INSQFCMM1bUN\nXLibdJqzuvkssrBepAVLp4465HwOU7fncGZ7t4PzcJkU/2qyFtu34UI6A94Vn5yCW8rl2G1MTYmM\nsofX6PtcCDYmrpTvRlreZJJwsfFcRLcDd5bdxHPYPreZ8Sx8jnacP8dsHp9jp40QIZlh9+sFCBH8\nGHfl7bqcEtkPOiMpRlBDUowwVNfW7mBF5Ppc1NVsofAsHmOXEovBpZSKkDudLRqXFEdzYpEuZG3R\noLy5jqk7uqqyxHMFPmee46K+2/eFe4kcHfc6yNIn4uxS2h24qeaAO87FSlgJxvJwe+1nvIp1RMGa\na7Hb64sG8U4Al7W6wR1TNrfxt5iYiWwed/SWbeWAUENSjKCGpBhhqDGS08WStt3i48ryePFgUCdd\nSizX954gFdBos9+/8OKpUK5vcvxhx/Cr0lmzSBz0ZAGvmU5xrDZaQSxRKuM7WBrlNIQ1QPyUiaQQ\n6i1UPXQi18EHXVEZkES851h8/Nx3kCZw4h3SOQnESB0HcdDiMrfQadbxmY7Ocmbbtfm59oPOSIoR\n1JAUIwzVta2vYqntR1xKKoml79oGu6XBQDYjhXsZLfOUv7YhUgg2v75t4f/lRHY5WgCXSGPJPP94\nnnQzPbxf4hmW3ckkL/ELoptbPs9Fad0uXFs8Fc02wy0VMjgu7tncccQSR6r3XE6BxCbxGcuG9s0W\nv1cvwBwy9wdcmHfhEt/Psh90RlKMoIakGEENSTHCUGOkhQWczY/eW1ss4OfGHtt3s4nl6DnRB2Du\n2BiNW11fwusV+Sr3wMHWRy6PWCed5Bhp7ihiK9lmxrIsq9fDUrsmzqDV9yJFehVxnszhrRrbxmvW\n29yfYOAhbVCrY3d+pM0phLSIb3o2F/ClReugehPP1Y70xikdRsoiM8HP6BX4YMZ+0BlJMYIakmKE\nWBAEnzxKUT4BnZEUI6ghKUZQQ1KMoIakGEENSTGCGpJiBDUkxQhqSIoR1JAUI6ghKUZQQ1KMoIak\nGEENSTGCGpJiBDUkxQhqSIoR1JAUI6ghKUZQQ1KMoIakGEENSTGCGpJiBDUkxQi/Ap8fzOKOSZZH\nAAAAAElFTkSuQmCC\n",
"text/plain": [
"<matplotlib.figure.Figure at 0x7f5aa180dcf8>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"def my_imshow(x, ax):\n",
" x_show = x - x.min()\n",
" if x_show.max() > 0:\n",
" x_show = x_show/x_show.max()\n",
" ax.imshow(x_show)\n",
" ax.axis(\"off\")\n",
"\n",
"#test\n",
"fig,ax = plt.subplots(figsize=(2,2))\n",
"axim = my_imshow(x_train_orig[0,...], ax)"
]
},
{
"cell_type": "code",
"execution_count": 153,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-22T15:58:40.272000Z",
"start_time": "2017-11-22T15:57:51.123496Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"gcn started\n",
"gcn finished in 2.85s\n",
"get pc started\n",
"get pc finished in 24.79s\n",
"zca started\n",
"zca finished in 21.29s\n"
]
}
],
"source": [
"import time\n",
"\n",
"def my_gcn(x, s=1, lmda=10, eps=1e-6):\n",
" # gcn normalization based on \n",
" # https://datascience.stackexchange.com/questions/15110/how-to-implement-global-contrast-normalization-in-python\n",
"\n",
" # replacement for the loop\n",
" x_avg = np.mean(x, axis=(1, 2, 3), keepdims=True)\n",
" x = x - x_avg\n",
"\n",
" # `su` is here the mean, instead of the sum\n",
" contrast = np.sqrt(lmda + np.mean(x**2, axis=(1, 2, 3), keepdims=True))\n",
"\n",
" x = s * x / np.maximum(contrast, eps)\n",
"\n",
" return x\n",
"\n",
"\n",
"def get_pc(x, eps=1e-6):\n",
"\n",
" flatx = np.reshape(x, (-1, np.prod(x.shape[-3:])))\n",
" sigma = np.dot(flatx.T, flatx) / flatx.shape[0]\n",
" u, s, _ = linalg.svd(sigma)\n",
" pc = np.dot(\n",
" np.dot(u, np.diag(1. / np.sqrt(s + eps))), u.T)\n",
"\n",
" return pc\n",
"\n",
"\n",
"def my_zca(x, pc):\n",
"\n",
" # zca implementation based on https://github.com/fchollet/keras/blob/master/keras/preprocessing/image.py\n",
"\n",
" flatx = np.reshape(x, (-1, np.prod(x.shape[-3:])))\n",
" whitex = np.dot(flatx, pc)\n",
" x = np.reshape(whitex, x.shape)\n",
"\n",
" return x\n",
"\n",
"\n",
"def my_pre(x_tup):\n",
" # Custom pre-processing steps\n",
"\n",
" x_train, x_test = x_tup\n",
" n_train = x_train.shape[0]\n",
"\n",
" #combine x's but be carefuly about CV bleeding\n",
" x_stack = np.concatenate((x_train, x_test))\n",
" \n",
" gcn_start = time.time()\n",
" print(\"gcn started\")\n",
" x_stack = my_gcn(x_stack)\n",
" gcn_end = time.time()\n",
" print(\"gcn finished in {:.2f}s\".format(gcn_end-gcn_start))\n",
" \n",
" pc_start = time.time()\n",
" print(\"get pc started\")\n",
" pc = get_pc(x_stack[:n_train, ...])\n",
" pc_end = time.time()\n",
" print(\"get pc finished in {:.2f}s\".format(pc_end-pc_start))\n",
" \n",
" zca_start = time.time()\n",
" print(\"zca started\")\n",
" x_stack = my_zca(x_stack, pc)\n",
" zca_end = time.time()\n",
" print(\"zca finished in {:.2f}s\".format(zca_end - zca_start))\n",
"\n",
" return (x_stack[:n_train, ...], x_stack[n_train:, ...])\n",
"\n",
"\n",
"x_train, x_test = my_pre((x_train_orig, x_test_orig))"
]
},
{
"cell_type": "code",
"execution_count": 154,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-22T15:58:45.159359Z",
"start_time": "2017-11-22T15:58:45.155905Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(50000, 32, 32, 3) (10000, 32, 32, 3)\n"
]
}
],
"source": [
"print(x_train.shape, x_test.shape)"
]
},
{
"cell_type": "code",
"execution_count": 155,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-22T16:00:18.051332Z",
"start_time": "2017-11-22T16:00:17.926119Z"
}
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAACICAYAAAABDZUdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJztnXmM3Od535+579mZvS/uLsldkktx\nKUqWSEqUROqwa0eS0bp2EiBpnMQI6iIN+k+aooH/cIEAAer8URRN0SJIa6PuFddubdmVbZmSddCi\naNMiKd7kcu97dndmZ2bnnukfBd7P+y4KMArIbQs8378e7vyO9/q9fL7P9XparZYoFAqFiIj3/3YD\nFArF/zvQDUGhUBjohqBQKAx0Q1AoFAa6ISgUCgPdEBQKhYFuCAqFwkA3BIVCYaAbgkKhMPDv5sv+\n/E++ZsIiC7GG+Xtw2+dcF/QTPVmNlI3saYWM3Cp4jLwdphtt5YrzrLL1W73Ke0LBppEbVdrS3OYd\nIiL+jjq/SQnZ22ZkbzVh5I5w3rn/3NVfGvlTr37RyNO/+IWR4yneWWjS32g+7DyrGmFcUtu0a6V/\nn5H7rfsnP7rj3L/vk6eNfHf+LSP/ye9/1SMPEF/586+aho4OPm/+fu+Nt53r9k+MGXnRS197Fu8Z\nORtl/oIla84T9FNEJG7dX8iyBkaeeMLIP37tG0Y+deRx5/71MnPYDDKH3mYOWSJGrq+7n443yjt9\nQdZZpcr/uf4g6yxcZv5yYXfNRa3fPHHmvOXhHcES/a3W3emrRnlPvEhbfv8r//i+86wagkKhMNAN\nQaFQGOwqZcgFUHM6qqhFW65mLK0yak4dzV4q9ZiRgx60n0QDVaoYCzrPqltypx+Vv1DpMrI/lTVy\nLejSl1oNlS3pTXJ/lfuTnrtGTofjzv0bd+aNfPG73zXyoZc/ZeT8wkdGDmd66Et7xnlWs0HbmvE9\nRq7euW3ka0/Q/97ANef+OxdvGbmv44GyBAe9dcbjzkVoS0eg7lx3LY2aHvtF1cjNvXuNXKkuGLkW\nQRWObHQ7z6p3rhi569iEke0x35hkLtJPPOrcXyvxW648Srt8zP9Wk8XoTbnrxB9hPBtZ1kk6sGbk\njI+F3rDWaaLqzkXJwxqqWusv5Oe6Vt16VphxERFJVfms1wMf7xNXDUGhUBjohqBQKAx2lTKkykUj\n59t5dbPp1mTwtFDNYjlUs0gMi28taHkiajwrkoVWiIhUJWDklR6sub5mwcjRHH8PeFFdRUSaVe5f\nSvCegI+/Bw9gfV6pLjn3f96XMvLPunqNXBa8FJ7ldiM3BrGexxfc6elJoHJP5vmtsa+fe5ZR0cMd\nX3buHz1+1sgfvDslDwtXM8zNiWdR/0sXXnSuiy/j6ajvw+OwnqFvY5Y6vLLB32sDrpfBN88YlgcY\n26I15p+fhhasdLrzVO6HgpRvM7d5a/3Ynh1v0F0nLWsNldtYW0XLGxVdsdaM1IxcDfFdiIgEgtvc\nI9AEr/WOko/7m36LV4tIPsg9qQ3X63Y/qIagUCgMdENQKBQGuiEoFAqDXbUhbHvh08FNi4P5XLdL\nq92yNeQtV1sVrp5swSGL9aiRK4M7akRabqP0puWeCXUasdwJH8vJpnN7Iot7qDsNh+sqWi7MX+Ia\n83pct+fVRw4aOVLjt/oS/Hlxe93I++q4E2XA9ceeKxGRONLCJRlOw1nX1weNfG/P192+3IXDDnYl\n5WHBfvbs3WUj53e0J1o8ZeSOtDXueebjXOCAkQ8PYAMI1uHZIiL3tueM3L+ECzJSY23YczF2zXWB\n+lvMYW87rt+1GGuuWIOP51OuDaFNcBUmLbuTlOhXeZjPrdJgXTYXaKOISNiyb215WAPeMG2OJKxI\n303XbibWs7e9rhv8flANQaFQGOiGoFAoDHaVMlQttTBYYS9qlHZEzeVoViuG+uOzkjuqFdTvRie0\notpy3Swj535o5KZvyMhvZ7h/36vPGDkchNaIiCTr0JdKBDW1eJ6kpaVB1PzYEq5REZFgP263gwUi\nBxeuo9Z6nuL93W3jRr50gSQfEZHSPihL99IlI1+4QaTlxN4TRl6NuNF0vqaVAHQuIQ8LtSuMYfwU\niUKhiOse67Zcbx/dINnreIT7b6RfNvLiPfp/7DjjJCIylaZv+fffM/KROOr/VOoRI2cXXbdrsa/D\nyP0fXTBy/XnuScZR7WsxIlVFRCpV+nbjh7z/dCfUItWYNfL0iZeM7Ot1k5uqedamL8R6brYs+mx9\nI9KCvoqI+KxvZqdL835QDUGhUBjohqBQKAx2lTIEK6h1tRaW2Eq7azEOFLHsBj38VpM09zRQRX1W\n0lHi9cvOs8pWPny6gcp3+gjPvfNTLP7poyTGiIikx/Yb+ezCT2hLeMDIh+pYfLf7XLV4KACFCHUR\nTRero8q++R+vGrn7Nw4jTxxznjVWR5W9eJtIu0cbqJxtt6ESlx6DPoiI9C1BLRKjp+RhITGKml/7\n6JyRl/pcOnbwQ2jCo1Z9ios++vbcCLQpN3HcyFM5t+7EBWsMXz181MihLmjJUJ25WNoxT35rDt+x\n5jawwftfHETN374z6dy/eYUEtdNnoBOxLGtu00c0ZOr1RZ71GTfRqtJgbXqsNR/0bBm57ucbqbW5\ntCC0bSUBVmrycaAagkKhMNANQaFQGOwqZYhtWoEZ/QRfhLJu8EQuhJU07kG1C+exuFaCVgkpqzSV\nd8IK7BGR8NSMkVf2oFr153nHwQME88yvuV6CXB9q9qZlzR/oQ/3zWQfmtnlIphERacsSQDTXhpV4\nsxO1cniCgJOZ968bufuQm8BzKwO1eTlNwM5mgOeuruLJeaxr2Lk/68GSXm64NO1BombRvHA/lOux\nTneeVyNQhnQ3CVp/q8Z8/uDW/zRyTyfW9NWbbtDW8AT3b6ShJq0agVF7csxfJeHOU8FKamvrweu1\nYM15rg25umOd2GsonifQbLHFmgtOsU7KE8yNvX5FRIJ++hYqWJ61BN9CwWutuawb2ORLWd/W4o6g\npftANQSFQmGgG4JCoTDYVcrgSVqVZnOor10eV30N+1CTAgXyyTfCWEx9VjmraJkyVVPHXYtt5ANk\n/w0CRrzDqOzV/iu0ccFVa1uTVOcdauKxKG+jMm5sYxUvxd2AleAB4ufv3eE9DR99jkc/YeTuzneM\nHJujNJqIyETPU0aeSmFlv1FFRezpIviqe+n7zv2Fo1j/621b8rBQPmVZw3OorG1X3PZsPkJbbzZX\njTweZA4mmvS5OIfHonsYyiQi0tx+zsiZ4g3eYVGjxAG8D2tT5D6IiJQKrKFatM/I9py3JqEini3K\nxImIVPuYA+8ia9s3w3MrYwSgzViBVXvnXc9YUwhMKoehL+3WtxALM8YejxvYtGZ9Wz3JgnwcqIag\nUCgMdENQKBQGu0oZ8l5U/jafleK8o2pss4zK5K9gWU54sbpXrTRj/5alYl6/6L7zESiEdxxVsPA9\n6EM4blGBfa5V9kKDnIWjR4lFv3gJ+tM7dMjIrxxH/RcRee21vzByOkKgUdFiJl0tgonuLqCW9g+c\ncZ410KIt2fdREydGCFLxHsCq//4M/RUR8f7n14x86IRFrZ6RB4r2D7Gy3/zgTSPPj7qHozzVi3W8\n+/aGkVvTeEpSx/ESzHaeMfLigkszR3sZwzXr4JNYFm/Q/GXyWl599fec+79/gXUzP3vTyAeO8Ylc\n2KSNQ/vceQ5vsZ4LM9CR7c9+wchN7/95nVbFDdiqC8+KNvB+5KzAvnqA9e9NuqncbXapwpYGJikU\nir8hdENQKBQGuiEoFAqDXbUhhKpwO49lT4hVXFdfwme5URIkQeU8RIpJxTrdycvfm3H3RJ+NLZJQ\nurdwCW78CnIqRHTj+CYuGxGRhWlKl69ukkR1oosSbPsO4AI7d+28c38hSLJUwA/XixetktpJ+GBP\nHM48kaJkmojIzRa5+X2fpC1rRe6fzxAZeTLnloPLHXvMyG/9iH79oz/4+/Ig8daPiKg8c4bS620L\nC851FzMk7gz2Y9PoGoM332ziXpvwWKXnc+6pViU/kYfxLVx1VWvMa9Zc7JynJw4wtu0lXNqL93iP\nT3A1j49gdxIRqe7BbpIdxtblXeRZmSRrMRBjnTYLuFz/9x9wLzZCVg0Mv2VrqPH95HNuDZCW13KB\nNl2X5P2gGoJCoTDQDUGhUBjsbgk1L+7FkA/3XqzLpQzvzJP3v7eJat8ZttR563Scao1yYBG/qyKt\nr1KqLJ+GWqzeRMV78RDuwE0vfxcRkQxqYtlvJV19ArXOlyG3vRVy+1IIo87FA6i/nXnGohil/dEG\n8t2Gq0reucP7K3Xk/KB1CKqfOgsBv+vObWVR2T998LA8LNjPLlnvDPjbnOsifvq6XIUqFe9BB2f9\nuNB8Y8jJhlsCrlKHWsbKRHEuJVCtCx5oVivkrhN7DuOD3FO+aCUx1aEiK/vcdRKusJ7fs9ZWN6UV\npLpKv/Z2QZdKYbcvwYAVXVjmuXMVyr5NVYmAfGaQOhEiIoUNq4SauOXV7gfVEBQKhYFuCAqFwmBX\nKUOunYi6x6wyT/Nr88516RrRdusBosMGR1HZaj8nOWQggWp9Wz50nrXfOiA2uEoyTbqFVftyBKv4\nac+oc/9mDovx+RG8CYMePBOrFdp4oOzusQuW+hqPoSZuBVDlZhOU6RrO08Zjq+6z/PtQq+tzUIPo\nJipuj4doxruHydEXEQktQWfqO3LwHyTybajjdT9q8t397uEm4/do60qLw14lgvp/xLLej63iSbjk\ndUugzVhjOJTnuqSVKOXNQxmHtlzKcNmidv4wc3AjwZo7OU2yWW+EORcReTsMzT1ora32VaIWqyna\nmCnTrwMVl75tZKiVEHiS+9c/wmNifyP5Hd/PYJjaEB+6LO2+UA1BoVAY6IagUCgMdpUyHLZqDdzp\nQOX76AO3HNVzv4sKvDFHMMZb38AS/MqvUsH42jz3RwJuaa1KDQpQaXHd1SvvGvlTR9CrOjMjzv2p\nUwT3PNdGmyMRLNYLHTz323fdPPuZbu5/JUIyj38blX38bd4ZJI5H3hh0VcGxn0A5Gl1Y3MMB1M+l\nEuPSLLrnTEabjGstvMOb8gDhsZ4drRLAVdrRnqWQVYU6Qk2Jcg1vUuAuFvg3XmI8xuZdOhQ9O2Jk\n3zhjU49DU96wqjkvNt3ydMMDUMCBCnP73GGr7sYAa2nnOpnovmPkH1tr67FPQDM9XhKiQgH6cq3u\nrv/xI9Dp71trfvwFxqt9Ao/VO//OTfQqPU9S2+EFPdtRoVD8DaEbgkKhMNhVyhBpoor94o1/a+Tn\nPvcl57pQyDpe/SbW4M7DqNyrZSy+kxsEaRy1LKwiIt5OPBvzZfLcT/8m6t9SDktysuSep1iI8rx+\ny0uS/AbVgIeskltHB1wvRXIFyvDDw6js7z4OtXk1zjQ8dgVVsmcJFVFE5INOqM1QAM9KpsX7OzxY\n6BsFty/VNqzvudbDO9sxawXatFnnSwYKi851UQ+5Ghkrb7/dokDTncT8P/Nz5qlzDY+LiMh7T0Ih\nXjvA2D57E8rwx9eZm60e6JeISOwya668zaEr4S+eMfJiDs/Q8o51spSnnad/k2C6Wx9CUwaDjL83\ngyo/WXbPmezoJDCp0/IUeVm+EhplXE981vVSvPOdvzTywWMfL09FNQSFQmGgG4JCoTDQDUGhUBjs\nqg0hfhRuN54imit/66xz3dwkHKw+S73Cjg74ddVyGx1vhxuuF90ach4ruWawl/0vn8I9WFsnSs43\n77pptk7hEmrk4aMvbsPnC508d2DLjWDzDuNC+p034YbpRdxGS69gp/huN3aDHvc8U+kv0pdslbHs\na9DG9V7cZInJEef+Yo3rwhW3jt+DRHiVZ1dDdCKx7bYnvx87SN8yY7jigx/3++nz3Taee26va1+p\n9WM7+ez3sSn97ZuM//YJbFCJVXeeClHe35thbq/lGbNiF2vOd84NAawlWRv5FC7IwV7mqWGtxWIL\n25i9fkVEqk0S8tp9rJPlWewJmXcwKKTqlnFBRMbP8G3Fh1w3/P2gGoJCoTDQDUGhUBjsKmU4O8Xr\nYjmSc/yD7qGkIctd5usl6my2QuLSoSaq/bSlPtc33ASazjiqZbwT1ezObWhKrWUdDjrhuqPSVVxj\nz/4Fh5PeenXEyD+oXDNyMu0mzeQitOdPxz5t5Oc8HEJ7+7/iQl07RgJ9ZY9bAq32Dq6ttj2o2/Nh\n+jg4A2Uo9rmRjuE8bl9f0H32g0TcenbDiypf29GejhnGen4P/zd1llHnc3NWqbHncEf2zLnz1GWN\n4YE48xweY239swHKsLe1u1GTW5skN728l3JqL/5rXKXv/p51itOEW8JtowJNyd9G5R9LEHqa3+Ad\nmQJ9zAfctvQ3edasVQOhvZc+++MlS+ZEKhGR4BLU5myDb+4luT9UQ1AoFAa6ISgUCoNdpQzZN1GT\nqk9hcd+30elcNz2Pml1ewjL7had/zchbUVTjtZ9z8GZq1N3jUk2oSXYdi20qyCGs0VDaugYVV0Qk\n2n7dyOk22v+eVQC6I0VueiM47dw/8BHq/D/p/7qRT6aeNfKLH9L/8DeJhnzjjDs9a3tRuQv3UIU7\nu6zEoCjvy266lvA+y3rfsMqBPWh46qjAIS/vXNrRHl+U/nVapyIV1qBN1X1Qq673sbJ/8qfuPJUn\nGMOzB6gbcD77dSPv+YiIvu3D1gSKSEeIOXwvSxTkSWvOoyXWQnbDqo0mIsNp1tB2hbWVX2c9ppKs\nRUmRkFS8u4MyjH7GyIcOcf+3vsXpVOEm30jvoOsZWwlCZ7atb07+SO4L1RAUCoWBbggKhcJgVynD\nC1/7KS/+ERb39aRr8R4ucqBIuQ9q8dYvsax35VHZOg/vNXIs6ua5T19CnT544CnuEWTb4tsagH6I\niIx+m8M1vvlZ1OzxPVi/x/Kostsl96CXwGPkw68nSG75L1nU2rE6be7sRl2t+OmXiEjEqmcQ7aNf\n+SqqeFJ4bjTg5tlvRanB4Ks+vOSmcpx2NoKWyr3tc67ziFVSzjqcJGb1Taw+V/x4STa63YSgjHXe\n6Q+tas6/PvqCkTt6WFc1wSshIhId4f13ElCzb44wHy/8gLUw93ddL0ffKh6IzgC0LWNVjb51+30j\njxyD8kVG3eCh2dfxVK0loMzDj1vUsjxi5FCvW07umS0Ck+pf+6H1yxfkflANQaFQGOiGoFAoDHaV\nMvynv7Bi+cPnjBxsc9W/T75MMMfK6qyR+1tY4HN1gpS8G6jpm1U3xj0cJRjkduZ/GPleF1bahZ9j\nfW4OElQiInJylEMwjkySc7AQREf99o3vGbnHogUiInUvgS0pH9ToyT1YtZsxrMxb7RyykUtberCI\nxCr8Vs+hPiZiWJXLHvofXnWtz9EIz9tsPTwvQ8N6drpl5RJsuMut3M2/Ey3LA2X1zWMdyGOPx9aY\n62VoVnjnk2FU7uU7qPk3rZwPf9M6DEVEVvJQg6PjnzXykUlo2+Qoc3n+slvbwTvP3Aw8hTeinmFu\nw1EC4Datg1bCYTevovmSVWnaDzWteQhG6ulm/byxIxeoukqbN61v7gv/Uu4L1RAUCoWBbggKhcJA\nNwSFQmGwqzaExx/FVeNdh98OHet2rrt9Hjdc5j2r9LhFz8MdcCivD54Ujrn2iNABosYGrbLfG1vY\nFk586QbvDrvJUek52nlu5QMjB+7RmBPdJ4y85XUP16zlsHsEU7zz8jacddBHAlU6SgKVJ+RGsDVz\nuN28ftxx9Qhuz7CVc59PuRGgjTgc2je7o9jCA4SvwLPzQ9gQyinXppEoY/uopazagRXub1bps8eK\nGtyMumXkN+u4/e5ukxx3PEQNhHTWKu/ext9FREYiuAGn7/3cyPM9zNOpHk4L69jj1nQ8YCdk3YTD\nt49ZLsQKbt9q5ZaRt7Kuq7q5ztqeX2eefcu4J3PP8I2cOul+P7ObjHOzw3WP3g+qISgUCgPdEBQK\nhcGuUoYDAfLUl0OoP/Nn3dzycPpnRn7xq6eNXLqJmrhcRRUr30MV2466EXhr2yTH3JiGWgw/StTc\nwNuoVauHXFWyusAQxctEvWWLPzByo5cot8COEm7RMQ4Ord/lcNMjyVeNvLdBvzZbRJ2Ft9zpifrZ\nvytVqE3Kinpc8dCvhLgRcFUraceXHpOHhab17EYWt19YepzrNqy29lh9yLbom91nscaj1XKj8/Y2\ncPtlvNCHQO41I0fGiOCrLUITRUQaMd5ZWoR2prwvG7lquQpXm5z0JCLy+E2SleYniHa9epm12dmH\nKp+y1mlHyT14N2wl6PVa7u3IIajFrXfe5n1nXVd3PMK/ewMfr1SeaggKhcJANwSFQmGwq5ThigfL\n6v5eqinPvO9GF87fQG2/+B4VZfsmsFj3WklPcpBuDO44hHNfyYr2O4xltnwNVWrmDHUSnq1+x7k/\nu/pnRk7mUT8feZzaDB+9j2cgnXYpS61wlX/McsLSTJjTgVZeIgHqi3+KZ+CnfwbdERE59jZqaqGF\nJb4YJ4Emtg1N6Gi4SWPLm4xfe2RNHhbaizw7U+KdHSG3PU0vbS3G6EPNqm6dqDGe108zHi/9oetB\n+cY/RR0vX2NsJ6YZ81qTuWg1XTq1PMN4nnqKuZ27fsHI2QQ079n2P3TunznzOSMPv4OX4aBdd2GR\nd1atWhAbB915LheJlj0/w//ZS99hXKtePFODabcvR57Cm3Wlyjf3a3J/qIagUCgMdENQKBQGu0oZ\nhjJYmZe9qPzHDroW271F1Plmk7z1yzNYrFNeLK5p6+DS2YarfqV6SIKpWqUSquN4A2oXaMtK75PO\n/Xum8CAkEr9l5PLVnxi5MYSFu9JwA34Cftq5fRT18eA8bd582UpUWuf+rowb5FTfxhod7EQV9xTs\nuge0dzXpJvBUt7mnVHITpx4kShU8ANUY478aLTrXeQuo+Z4SfQim6EM9Q5/t8bDHSUQkOcIYDr9l\n1Yo4ivocrRPkVKu7gU2NBHNYvko9hF5hzmWKfs2l3HVSX7Y8YOO8M1ikzcEeqGl2lfXXlXNp5qaH\ndbI+gwfuyQkrGM8LzWyLueXg5pepGzLUdD0794NqCAqFwkA3BIVCYbCrlCFpxfx4KlZprJJ7gEfN\nT9CRpw0PxItJ8hI2GuSjZ7zEka/F3Tz1olXRuctSC30l1NX+U5e5fnPCuT/yPPkLN25jzb1ynr+P\n78VjshVxPSb96wQDXRyhnS8MUQKuuAAVWhvg/s7rrrq3lUTN78oR1JMPWupuDRW1o+xWOW7UUGXL\nDbeewIOE/exAjTbHdrRnvWXloPjpQ5vVtzWrz53Xmf+1ATcALGiNYfsQ6+fNImP+9LT17g63anKo\nxHU/mGJuj56E8o0/AedMeV3Lfmw/a2hl8jl+CNDHhTw1G7YHoAKeAh4DEZG0FfT04gR5MqshvAyN\nHPO88/vp9POehBtnd1+ohqBQKAx0Q1AoFAa7ShmulqxjrnvIH8gXupzrUtWLRu4NYhmf2oAOtAvx\n2okKKqp3zT3Bzuu5YuRCCCtzs5sju6fyxN4vXnCH5NCnvmbkoWc+ZeTTqd828sUWlW2LFTf9unOU\nd/Z2Pm7k5/45Fve/88cEjzw1blmS21zrcaCFxX2jzF7e3YJmrHj5+1bDLc1Vb+e3QN5NrX6Q8MQJ\nLKonkLe2tp3rvF7abZdaW/VwXSCGZd9r5T5sj7sx+v+tziEm//2voBbX/4gx7+whYCm/RZq4iEjM\nOqznDw7+tpGnjhCMNlv/sZFv/phAJBGR/qdZQ60O1pZ3lVyW1BZehmSOdRrzzznPCobo56xARfYm\nmP/lAOtnITji3J+IQy1mVj7eGZ6qISgUCgPdEBQKhYFuCAqFwmBXbQhLe+BtXitQrBlx+VwzjUtw\ncgU+FT2MC6c0TQSjrx03TVvOLS8eS1CaazMD7+qoUpvhl1dw4fQPwuFFRDLXya2/nOC6l8/i6omX\nKGG1//ivOPcnf0w5r65nef/FU/C8zzdwB97sIwKvteVGcCaDROBt+3GBVUO2rQA3ayDjnmIVWqdM\nWDLtljN7kEg1cOl5p5izZtiNIm1Y+UlVoQ9Bwb4RreDeK1dJVLrZ57qX7TG0x7brGtcMvPuIkRMn\nXVffygUS1JYjzO2bMeZ8IM9a6I8QKSgikn0Nt+fjR7FvrVu1DtL9rMWi5YJsteEmFREplVnb4RH+\nz568x5xFenDH1krYT0REsnl8jfY399eBaggKhcJANwSFQmGwq5Th9ALn3uc3cBWup3YkBLVwtawX\nTxp57YM3jTxw1ArB+iW6ZzHlqk/VJdxZwbjldvTh8urtw+2XfsI97PXGm9CB4Sju0c3fsJJZlmj/\nyiQUQUTE+xilrg5O4La8u4qrafg1XFbZMSuxp+aq/C2ranQkgsqcE1xwPWu46Ta73Gi67RV+83kf\nntuxaD27ELRKwu1oT0+G39a66EMygtuxJPTZHg9/wI3iHH6NZ8+dJArwYDfP/VkBt2ETD6CIiDRe\ntio990FThz+EWmxtQ0WGX3DXSdiiLE0f1Cxo0bnNJeR8mPmPTbrRsfI4ruuFDxiLmB8XarLKaU1B\nzzHn9o4syVKj5aPycaAagkKhMNANQaFQGOwqZZj2oQr5RlArYxk3t7yU5reORdSscIrSVOFpKEfe\nQwRkLMDBKCIirbBVDyFl1S1YQK0M9KCKZu+4h14cG8cCXIqgSl5Jo7KNdJ3hmkFKbomIxFNHjDwZ\n/TbvvEE16dXVp43cMUzF6VK7ayEOTTFOzWHanNxEXd1IjnDNhqvWdneRaFXdbMnDgqeEat3dxdxs\nbrjJTXZbk1usDU8amhCbgQ5W9nKASmTDTU5aXWU8Q+uM4eSjjHn0008ZeSvrzlO0Ha/TlTpze+TQ\n80beVyKaceWOS38CPRxKvFGwEpcGLG+Cn7UYa7BOax73gOHENN6IjgBJXLEs30Kpj7UQ2+R6EZHs\nCOO0XnQPHrofVENQKBQGuiEoFAqDXaUMoQ4s44kUARv5opvcEehGndq8hzrdH4IyFNrwODQKI0aO\nx9yyY8US3ojIIsEcuQqJQ+FhK+BjyT0AZM7KLU+vo9odvITHJB5HrVtud+shVL5nBU29wkEt5f20\nM3mXytLFvHUYS4ebzL7VhmmfBjDKAAAEq0lEQVQ8bpfmSlhtzOBlmdt07w8k6acv6XowHiQa1rML\nTd65ve6O7Z4mba124gGqWH0rWH0OeOhPM+/+X5bsYQzr1tjG5qF8W99nLvzH3KSfjkt8CqECc9vY\nQzDVXBLK0CZuXyqdVtXnDZ7dtoiXoxSh/bEY67+ccwOTym3TRu5scjjQYpA1m+qGFtXKLv1J90IT\n8lmffByohqBQKAx0Q1AoFAa7ShnqU68YuTz6V0YO1A4713kbHxq5v4NY+FXraOzlEtecPEIgyJUp\nt4bAWAB1erIbK/tQAcvswkUC3lOJZ537wxtUA651oxYGnsUDkPERpBQbcAN+akexWN+bJBa+vQer\netdBgpnqy5TMCn3gjoucJp/fewv1NXIQFf3aChV3U7NuwErwkXNGXr7g1iZ4kFi7Rt96j6MmB2ZP\nOdfdTJ438iNteHfqt7DgBw4SMBR6m7nxdk07z4ocfJf3R5iDxRXGvOe3mItwm+slyCwwh9EGZdNq\ns8x5+B7/f5YDbi5IdpH1ODBMzsRsnHbu3yLH4U6OdXr0iOt9OX+NZ/WWCVrq77BL4DHPgdpjzv3l\nPPfXp35VPg5UQ1AoFAa6ISgUCgPdEBQKhcGu2hAiUZKIfLNwZX/BzS2fuYrdIN0Pb85miE7rKv06\n12eoEzDSDs8SEbm3hdulZ4FaBz7LVdgK/wMjX76EbUNE5PBn4H2rdSLQeqz6kDJL0pPnMvX8REQ2\n4kQL7vfBG5facZNtHTtu5O1fUDPh6RXqQYqIXDpL0srMv6Gdp78F541VGdeta26p9dDfe9TIx74c\nk4eFY1+mXuDULP3Z2Z7YAWw/HULp86tfgc8PfRkOfMzLePys03WntY4R0ViO4YbbP42tZfaydVhw\ngchEEREJMB/lIWobrPhJokoNsRauv37duX3oEdaQr/Geke01dy+OrcdepzMZ1wVqr+2ssLYDKexW\nm1dJiBtuEA0rIlJvYEOxv7m/DlRDUCgUBrohKBQKg12lDIuLVqJFkAi0ti5XlYwuooLPvQ5NGH8a\n9XN4lGi085ehFf6+V51nhWuoorE4LpzaFOrqvmf+g5FHdxw8W14mCcgudZX9Hkkv/iOo8pUkJbhF\nRIbKnOo0WYMyBGL0JXeBsei6fcbIvrKrSs4+S3RaJNlv5I1LlBZb/le0a+B3/71zf+lRpnsumbF+\n+YI8SMwNU5a+lEaVj7/oHjC78C9+x8gb/xAKFvkS4zx7grn9xLtEraZuQwVERNYilB2rPc/YTm5B\nE/YHWEtrSVzQIiKNPIlP9e/h6mx73jpgOMdaeOFzbhJcM8waqrz3SSPH9tKvcI21kW9C7UqrlHoX\nETn5FNGNM3ehRjdeHzdy936+pY1+t/R/bg2XpFRd9+r9oBqCQqEw0A1BoVAYeFqth5cXr1Ao/v+C\naggKhcJANwSFQmGgG4JCoTDQDUGhUBjohqBQKAx0Q1AoFAa6ISgUCgPdEBQKhYFuCAqFwkA3BIVC\nYaAbgkKhMNANQaFQGOiGoFAoDHRDUCgUBrohKBQKA90QFAqFgW4ICoXCQDcEhUJhoBuCQqEw0A1B\noVAY6IagUCgMdENQKBQGuiEoFAqD/wVDbNnVKpzB5AAAAABJRU5ErkJggg==\n",
"text/plain": [
"<matplotlib.figure.Figure at 0x7f5aa18bb470>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"#test\n",
"fig,axes = plt.subplots(ncols=2, figsize=(4,2))\n",
"my_imshow(x_train[0,...], axes[0])\n",
"my_imshow(np.flip(x_train, axis=2)[0,...], axes[1])"
]
},
{
"cell_type": "code",
"execution_count": 168,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-22T17:35:23.285784Z",
"start_time": "2017-11-22T17:35:23.282541Z"
}
},
"outputs": [],
"source": [
"# add flipped images to training set\n",
"x_train = np.concatenate((x_train, np.flip(x_train, axis=2)))\n",
"y_train = np.concatenate((y_train, y_train))"
]
},
{
"cell_type": "code",
"execution_count": 169,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-22T17:35:52.022216Z",
"start_time": "2017-11-22T17:35:52.018607Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(100000, 32, 32, 3) 2.4576 (100000, 1)\n"
]
}
],
"source": [
"print(x_train.shape, x_train.nbytes/1e9, y_train.shape)"
]
},
{
"cell_type": "code",
"execution_count": 220,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-27T15:12:19.496610Z",
"start_time": "2017-11-27T15:12:19.468627Z"
}
},
"outputs": [],
"source": [
"def top_3(y_true, y_pred):\n",
" return keras.metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=3)\n",
"\n",
"def plot_history(history):\n",
"\n",
" measures = np.unique([m.replace('val_', '') for m in history.history.keys()])\n",
" num_meas = len(measures)\n",
" x = arange(len(history.history[measures[0]]))\n",
" fix, axes = subplots(nrows=num_meas,ncols=1,squeeze=True, sharex=True,figsize=(6,2*num_meas), tight_layout=True)\n",
" if num_meas == 1:\n",
" axes = [axes]\n",
" for i,meas in enumerate(measures):\n",
" if meas == \"lr\":\n",
" continue\n",
" axes[i].plot(x, history.history[meas], label=meas)\n",
" if \"val_\"+meas in history.history.keys():\n",
" axes[i].plot(x, history.history[\"val_\"+meas], label=\"val_\"+meas)\n",
" axes[i].legend()\n",
" if meas in [\"acc\", \"top_3\"]:\n",
" axes[i].set_ylim((-0.01,1.01))\n",
" axes[-1].set_xlabel(\"epoch\");"
]
},
{
"cell_type": "code",
"execution_count": 215,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-27T15:08:15.921546Z",
"start_time": "2017-11-27T15:08:15.910824Z"
}
},
"outputs": [],
"source": [
"def comb_hist(old_hist, new_hist, lr=None):\n",
" \n",
" if not old_hist:\n",
" new_hist.history[\"lr\"] = [lr]*len(new_hist.history[\"loss\"])\n",
" return new_hist\n",
" \n",
" for key in new_hist.history.keys():\n",
" prev = old_hist.history[key]\n",
" new_hist.history[key] = prev + new_hist.history[key]\n",
" new_hist.history[\"lr\"] = old_hist.history[\"lr\"] + [lr]*len(new_hist.history[\"loss\"])\n",
" \n",
" return new_hist"
]
},
{
"cell_type": "code",
"execution_count": 237,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-27T15:32:20.903655Z",
"start_time": "2017-11-27T15:32:20.819906Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"input_71 (InputLayer) (None, 32, 32, 3) 0 \n",
"_________________________________________________________________\n",
"flatten_82 (Flatten) (None, 3072) 0 \n",
"_________________________________________________________________\n",
"dense_167 (Dense) (None, 1000) 3073000 \n",
"_________________________________________________________________\n",
"dropout_277 (Dropout) (None, 1000) 0 \n",
"_________________________________________________________________\n",
"dense_168 (Dense) (None, 750) 750750 \n",
"_________________________________________________________________\n",
"dropout_278 (Dropout) (None, 750) 0 \n",
"_________________________________________________________________\n",
"dense_169 (Dense) (None, 500) 375500 \n",
"_________________________________________________________________\n",
"dropout_279 (Dropout) (None, 500) 0 \n",
"_________________________________________________________________\n",
"dense_170 (Dense) (None, 100) 50100 \n",
"=================================================================\n",
"Total params: 4,249,350\n",
"Trainable params: 4,249,350\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
}
],
"source": [
"dropout_prob = 0.5\n",
"model1 = keras.Sequential([\n",
" keras.layers.InputLayer(input_shape=x_test[0, ...].shape),\n",
" keras.layers.Flatten(),\n",
" keras.layers.Dense(1000, activation=\"elu\"),\n",
" keras.layers.Dropout(dropout_prob),\n",
" keras.layers.Dense(750, activation=\"elu\"),\n",
" keras.layers.Dropout(.8*dropout_prob),\n",
" keras.layers.Dense(500, activation=\"elu\"),\n",
" keras.layers.Dropout(.5*dropout_prob),\n",
"\n",
" #Class\n",
" keras.layers.Dense(num_class, activation=\"softmax\"),\n",
"])\n",
"model1.summary()"
]
},
{
"cell_type": "code",
"execution_count": 238,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-27T15:35:58.820419Z",
"start_time": "2017-11-27T15:32:22.183265Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"learning rate: 0.0031622776601683794\n",
"Train on 100000 samples, validate on 10000 samples\n",
"Epoch 1/100\n",
"100000/100000 [==============================] - 10s 101us/step - loss: 4.6538 - acc: 0.0448 - top_3: 0.1024 - val_loss: 4.3438 - val_acc: 0.0644 - val_top_3: 0.1416\n",
"Epoch 2/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 4.2536 - acc: 0.0795 - top_3: 0.1673 - val_loss: 4.2382 - val_acc: 0.0699 - val_top_3: 0.1533\n",
"Epoch 3/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 4.1171 - acc: 0.0933 - top_3: 0.1909 - val_loss: 4.2038 - val_acc: 0.0779 - val_top_3: 0.1586\n",
"Epoch 4/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 4.0315 - acc: 0.1030 - top_3: 0.2069 - val_loss: 4.0885 - val_acc: 0.0829 - val_top_3: 0.1778\n",
"Epoch 5/100\n",
"100000/100000 [==============================] - 6s 58us/step - loss: 3.9631 - acc: 0.1129 - top_3: 0.2214 - val_loss: 4.1374 - val_acc: 0.0816 - val_top_3: 0.1732\n",
"Epoch 6/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.9269 - acc: 0.1172 - top_3: 0.2267 - val_loss: 4.2826 - val_acc: 0.0689 - val_top_3: 0.1569\n",
"Epoch 00006: early stopping\n",
"learning rate: 0.00042169650342858224\n",
"Train on 100000 samples, validate on 10000 samples\n",
"Epoch 7/100\n",
"100000/100000 [==============================] - 10s 100us/step - loss: 3.5944 - acc: 0.1578 - top_3: 0.2905 - val_loss: 3.7844 - val_acc: 0.1295 - val_top_3: 0.2514\n",
"Epoch 8/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.4576 - acc: 0.1775 - top_3: 0.3196 - val_loss: 3.7639 - val_acc: 0.1349 - val_top_3: 0.2543\n",
"Epoch 9/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.3771 - acc: 0.1892 - top_3: 0.3378 - val_loss: 3.7243 - val_acc: 0.1435 - val_top_3: 0.2707\n",
"Epoch 10/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.3029 - acc: 0.2002 - top_3: 0.3545 - val_loss: 3.7347 - val_acc: 0.1422 - val_top_3: 0.2673\n",
"Epoch 11/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.2520 - acc: 0.2071 - top_3: 0.3651 - val_loss: 3.7254 - val_acc: 0.1457 - val_top_3: 0.2736\n",
"Epoch 00011: early stopping\n",
"learning rate: 5.623413251903491e-05\n",
"Train on 100000 samples, validate on 10000 samples\n",
"Epoch 12/100\n",
"100000/100000 [==============================] - 10s 100us/step - loss: 3.1687 - acc: 0.2200 - top_3: 0.3845 - val_loss: 3.7070 - val_acc: 0.1494 - val_top_3: 0.2820\n",
"Epoch 13/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.1491 - acc: 0.2251 - top_3: 0.3911 - val_loss: 3.7071 - val_acc: 0.1506 - val_top_3: 0.2844\n",
"Epoch 14/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.1430 - acc: 0.2263 - top_3: 0.3903 - val_loss: 3.7050 - val_acc: 0.1503 - val_top_3: 0.2869\n",
"Epoch 15/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.1295 - acc: 0.2280 - top_3: 0.3956 - val_loss: 3.7026 - val_acc: 0.1501 - val_top_3: 0.2870\n",
"Epoch 16/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.1197 - acc: 0.2299 - top_3: 0.3946 - val_loss: 3.6998 - val_acc: 0.1521 - val_top_3: 0.2864\n",
"Epoch 17/100\n",
"100000/100000 [==============================] - 6s 58us/step - loss: 3.1082 - acc: 0.2332 - top_3: 0.4004 - val_loss: 3.6989 - val_acc: 0.1524 - val_top_3: 0.2873\n",
"Epoch 18/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.1025 - acc: 0.2330 - top_3: 0.3985 - val_loss: 3.6958 - val_acc: 0.1531 - val_top_3: 0.2885\n",
"Epoch 19/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0885 - acc: 0.2351 - top_3: 0.4029 - val_loss: 3.7055 - val_acc: 0.1518 - val_top_3: 0.2871\n",
"Epoch 20/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0831 - acc: 0.2372 - top_3: 0.4053 - val_loss: 3.6967 - val_acc: 0.1560 - val_top_3: 0.2889\n",
"Epoch 00020: early stopping\n",
"learning rate: 7.498942093324558e-06\n",
"Train on 100000 samples, validate on 10000 samples\n",
"Epoch 21/100\n",
"100000/100000 [==============================] - 10s 101us/step - loss: 3.0651 - acc: 0.2415 - top_3: 0.4094 - val_loss: 3.6958 - val_acc: 0.1558 - val_top_3: 0.2892\n",
"Epoch 22/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0686 - acc: 0.2384 - top_3: 0.4072 - val_loss: 3.6959 - val_acc: 0.1557 - val_top_3: 0.2891\n",
"Epoch 23/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0659 - acc: 0.2403 - top_3: 0.4110 - val_loss: 3.6966 - val_acc: 0.1557 - val_top_3: 0.2897\n",
"Epoch 00023: early stopping\n",
"learning rate: 1e-06\n",
"Train on 100000 samples, validate on 10000 samples\n",
"Epoch 24/100\n",
"100000/100000 [==============================] - 10s 102us/step - loss: 3.0709 - acc: 0.2398 - top_3: 0.4083 - val_loss: 3.6963 - val_acc: 0.1556 - val_top_3: 0.2895\n",
"Epoch 25/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0612 - acc: 0.2409 - top_3: 0.4093 - val_loss: 3.6961 - val_acc: 0.1557 - val_top_3: 0.2899\n",
"Epoch 26/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0591 - acc: 0.2409 - top_3: 0.4107 - val_loss: 3.6959 - val_acc: 0.1560 - val_top_3: 0.2907\n",
"Epoch 27/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0586 - acc: 0.2415 - top_3: 0.4112 - val_loss: 3.6958 - val_acc: 0.1556 - val_top_3: 0.2901\n",
"Epoch 28/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0637 - acc: 0.2401 - top_3: 0.4095 - val_loss: 3.6958 - val_acc: 0.1561 - val_top_3: 0.2900\n",
"Epoch 29/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0616 - acc: 0.2398 - top_3: 0.4086 - val_loss: 3.6955 - val_acc: 0.1562 - val_top_3: 0.2899\n",
"Epoch 30/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0580 - acc: 0.2420 - top_3: 0.4118 - val_loss: 3.6955 - val_acc: 0.1558 - val_top_3: 0.2900\n",
"Epoch 31/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0630 - acc: 0.2420 - top_3: 0.4100 - val_loss: 3.6955 - val_acc: 0.1561 - val_top_3: 0.2896\n",
"Epoch 32/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0658 - acc: 0.2399 - top_3: 0.4092 - val_loss: 3.6956 - val_acc: 0.1559 - val_top_3: 0.2896\n",
"Epoch 33/100\n",
"100000/100000 [==============================] - 6s 59us/step - loss: 3.0670 - acc: 0.2406 - top_3: 0.4072 - val_loss: 3.6955 - val_acc: 0.1562 - val_top_3: 0.2903\n",
"Epoch 00033: early stopping\n"
]
}
],
"source": [
"lr_sched = np.logspace(-2.5,-6,5)\n",
"\n",
"old_hist = None\n",
"curr_epoch = 0\n",
"\n",
"for lr_curr in lr_sched:\n",
" print(\"learning rate: {:.7f}\".format(lr_curr))\n",
" \n",
" model1.compile(\n",
" loss='sparse_categorical_crossentropy',\n",
" optimizer=keras.optimizers.Nadam(lr=lr_curr),\n",
" metrics=['accuracy', top_3])\n",
"\n",
" new_hist = model1.fit(\n",
" x_train,\n",
" y_train,\n",
" initial_epoch=curr_epoch,\n",
" batch_size=320,\n",
" epochs=100,\n",
" verbose=1,\n",
" validation_data=(x_test, y_test),\n",
" callbacks=[keras.callbacks.EarlyStopping(patience=2,verbose=1)])\n",
" \n",
" old_hist = comb_hist(old_hist,new_hist,lr=lr_curr)\n",
" curr_epoch = len(old_hist.history[\"loss\"])"
]
},
{
"cell_type": "code",
"execution_count": 239,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-27T15:36:02.800007Z",
"start_time": "2017-11-27T15:36:02.288956Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/lib/python3.5/dist-packages/matplotlib/figure.py:1999: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.\n",
" warnings.warn(\"This figure includes Axes that are not compatible \"\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAI4CAYAAAA/PH0eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xl4XHXd///ne5asbZO0TReabkCh\nLXSDUBBkkbKjFFQsKN6CAvfXW0DxdkH0ckFvb7ff7a23fFFEtAUUsLdL9QsiQrGySoptgVagtLRN\n6ZIuSZdss7x/f5xJMkmTJs1Mm0nyelzXuc5+zvtM2nnNOfOZc8zdERERyTWhvi5ARESkMwooERHJ\nSQooERHJSQooERHJSQooERHJSQooERHJSQooERHJSd0GlJnda2bbzeyVLuabmf3QzNaa2SozOylt\n3kfM7I1U95FsFi4iIgNbT86gfgFcdJD5FwNTUt2NwF0AZjYc+ApwKjAX+IqZlWVSrIiIDB6R7hZw\n92VmNukgi8wHFnlwS4rnzazUzMYC5wCPu/suADN7nCDofnWw/Y0cOdInTTrY7kREpD9bvnz5Dncv\n7265bgOqB8YBm9LGq1PTupp+ADO7keDsiwkTJlBVVZWFskREJBeZ2YaeLJcTjSTc/W53r3T3yvLy\nbkNVREQGgWwE1GZgfNp4RWpaV9NFRES6lY2AWgL8S6o132lAnbtvAR4DLjCzslTjiAtS00RERLrV\n7XdQZvYrggYPI82smqBlXhTA3X8MPAJcAqwF6oHrUvN2mdnXgRdTm7qjpcGEiMhAEIvFqK6uprGx\nsa9LyUkFBQVUVFQQjUZ7tb7l2vOgKisrXY0kRKQ/WL9+PUOHDmXEiBGYWV+Xk1PcnZ07d7J3714m\nT57cbp6ZLXf3yu62kRONJERE+qPGxkaFUxfMjBEjRmR0dqmAEhHJgMKpa5m+NgooERHJSQooERHJ\nSQooERHJSdm41ZGIyKD3tT+8yuq392R1m9OPGsZX3nNCt8tdfvnlbNq0icbGRj75yU9y44038qc/\n/Ynbb7+dRCLByJEjeeKJJ9i3bx8333wzVVVVmBlf+cpXeN/73pfVmrNJASUi0s/de++9DB8+nIaG\nBk455RTmz5/PDTfcwLJly5g8eTK7dgU/Qf36179OSUkJL7/8MgC7d+/uy7K7pYASEcmCnpzpHC4/\n/OEP+e1vfwvApk2buPvuuznrrLNaf380fPhwAP7yl7/w4IMPtq5XVpbbT0DSd1AiIv3YU089xV/+\n8heee+45Vq5cyZw5c5g9e3Zfl5UVCigRkX6srq6OsrIyioqK+Oc//8nzzz9PY2Mjy5YtY/369QCt\nl/jOP/987rzzztZ1c/0SnwJKRKQfu+iii4jH40ybNo3bbruN0047jfLycu6++27e+973MmvWLBYs\nWADAl770JXbv3s2JJ57IrFmzWLp0aR9Xf3D6DkpEpB/Lz8/n0Ucf7XTexRdf3G58yJAhLFy48EiU\nlRU6gxIRkZykgBIRkZykgBIRkZykgBIRkZykgBIRkZykgBIRkZykgBIRkZzUo4Ays4vM7DUzW2tm\nt3Uy//tmtiLVvW5mtWnzEmnzlmSzeBER6bkhQ4b0dQmHpNsf6ppZGLgTOB+oBl40syXuvrplGXe/\nNW35m4E5aZtocPeBcWMoEZGuPHobbH05u9scMwMu/lZ2t9mP9OQMai6w1t3XuXsz8CAw/yDLXw38\nKhvFiYhI12677bZ299b76le/yje+8Q3mzZvHSSedxIwZM/j973/fo23t27evy/UWLVrEzJkzmTVr\nFh/+8IcB2LZtG1dccQWzZs1i1qxZPPvss9k9OAB3P2gHvB+4J238w8CPulh2IrAFCKdNiwNVwPPA\n5V2sd2NqmaoJEya4iEh/sHr16j7d/0svveRnnXVW6/i0adN848aNXldX5+7uNTU1fswxx3gymXR3\n9+Li4i63FYvFOl3vlVde8SlTpnhNTY27u+/cudPd3T/wgQ/497//fXd3j8fjXltb2+l2O3uNgCrv\nJnvcPev34rsKWOzuibRpE919s5kdDTxpZi+7+5sdQvJu4G6AyspKz3JNIiID0pw5c9i+fTtvv/02\nNTU1lJWVMWbMGG699VaWLVtGKBRi8+bNbNu2jTFjxhx0W+7O7bfffsB6Tz75JFdeeSUjR44E2p4t\n9eSTT7Jo0SIAwuEwJSUlWT++ngTUZmB82nhFalpnrgI+kT7B3Ten+uvM7CmC76fePHBVERE5VFde\neSWLFy9m69atLFiwgAceeICamhqWL19ONBpl0qRJNDY2drud3q53OPXkO6gXgSlmNtnM8ghC6IDW\neGY2FSgDnkubVmZm+anhkcAZwOqO64qISO8sWLCABx98kMWLF3PllVdSV1fHqFGjiEajLF26lA0b\nNvRoO12td+655/LrX/+anTt3Am3Plpo3bx533XUXAIlEgrq6uqwfW7cB5e5x4CbgMWAN8LC7v2pm\nd5jZZWmLXgU8mLq+2GIaUGVmK4GlwLc8rfWfiIhk5oQTTmDv3r2MGzeOsWPH8qEPfYiqqipmzJjB\nokWLmDp1ao+209V6J5xwAl/84hc5++yzmTVrFp/+9KcB+MEPfsDSpUuZMWMGJ598MqtXZ/+t3drn\nSd+rrKz0qqqqvi5DRKRba9asYdq0aX1dRk7r7DUys+XuXtndurqThIiI5CQ9UVdEZBB5+eWXW3/L\n1CI/P58XXnihjyrqmgJKRCQD7o6Z9XUZPTZjxgxWrFhxRPaV6VdIusQnItJLBQUF7Ny5M+M34oHI\n3dm5cycFBQW93obOoEREeqmiooLq6mpqamr6upScVFBQQEVFRa/XV0CJiPRSNBpl8uTJfV3GgKVL\nfCIikpMUUCIikpMUUCIikpMUUCIikpMUUCIikpMUUCIikpMUUCIikpMUUCIikpMUUCIikpMUUCIi\nkpMUUCIikpMUUCIikpMUUCIikpN6FFBmdpGZvWZma83stk7mX2tmNWa2ItVdnzbvI2b2Rqr7SDaL\nFxGRgavbx22YWRi4EzgfqAZeNLMl7r66w6IPuftNHdYdDnwFqAQcWJ5ad3dWqhcRkQGrJ2dQc4G1\n7r7O3ZuBB4H5Pdz+hcDj7r4rFUqPAxf1rlQRERlMehJQ44BNaePVqWkdvc/MVpnZYjMbfyjrmtmN\nZlZlZlV6MqWIiED2Gkn8AZjk7jMJzpIWHsrK7n63u1e6e2V5eXmWShIRkf6sJwG1GRifNl6RmtbK\n3Xe6e1Nq9B7g5J6uKyIi0pmeBNSLwBQzm2xmecBVwJL0BcxsbNroZcCa1PBjwAVmVmZmZcAFqWki\nIiIH1W0rPnePm9lNBMESBu5191fN7A6gyt2XALeY2WVAHNgFXJtad5eZfZ0g5ADucPddh+E4RERk\ngDF37+sa2qmsrPSqqqq+LkNERA4TM1vu7pXdLac7SYiISE5SQImISE5SQImISE5SQImISE5SQImI\nSE5SQImISE5SQImISE5SQImISE5SQImISE5SQImISE5SQImISE5SQImISE5SQImISE5SQImISE5S\nQImISE5SQImISE5SQImISE5SQImISE5SQImISE7qUUCZ2UVm9pqZrTWz2zqZ/2kzW21mq8zsCTOb\nmDYvYWYrUt2SbBYvIiIDV6S7BcwsDNwJnA9UAy+a2RJ3X5222D+ASnevN7OPA98BFqTmNbj77CzX\nLSIiA1xPzqDmAmvdfZ27NwMPAvPTF3D3pe5enxp9HqjIbpkiIjLYdHsGBYwDNqWNVwOnHmT5jwGP\npo0XmFkVEAe+5e6/67iCmd0I3AgwYcKEHpQkItmWTDr1sQT1zXHqmxLUN6eG0/rxhJMfDZEfCZEf\nCQf9aNAviKZNi4QBiCWTxBNOLJEknnTirX1vnRdPJIMCDEJmGGBmmNHpcCLpJN1b+8kkJNxJJtOm\nuePes+N2wB0cJ+ngqXWD7QR9Un0HwmaEQ21dJGSEUv1gPEQoFCzXvvaWPXY4NoJtx1KvU3MiSSwe\nvE6xRJLmeJJYwokng+FDEQkZ0UiIaChENBLUFg2HyEsbjoaDuluOvbN+62sKjCzOZ0ZFySHV0Vs9\nCageM7NrgErg7LTJE919s5kdDTxpZi+7+5vp67n73cDdAJWVlT38ZyWSO5LJ4A03lnrDjSWcxliC\npniCxliytd8YS+unpsVSb9At71/Bm5e1DtM63dr2Ew/evNKHgzewzodjcQ/e+BJtgdGcGm5OJKlv\njtMYO7Q3PxmcLpg+mrv/pfKI7KsnAbUZGJ82XpGa1o6ZnQd8ETjb3Ztaprv75lR/nZk9BcwB3uy4\nvkhPNMeT7G+Ks68pzt7GoL+vKcbexjj7mxLsa4qxrzHO/uYEidQn6vRP163DTuu0eNJJJNs+2cdT\nQZNIfYJtWabljT6eHkSps4JkH3ysavn0nhcOBZ+Sw8Gn4rxIKPjknJqel5o+LC9KNG1627BRlBeh\nMBqmOD9MYV6E4rwwRXlhivIi7fqRsNEcT9LU0sUSNKb6rdNSwQu01hQJW+tw8Ik9NS3Vh7azGPcO\nw7SdxThOqOUMxoIzl2Cc1ukhS3WH0EbZMELWdrYWbCPoA6n9BMsmHRKpfydJ99Z/Ny3/vhKp8Zaz\nuJbjgJazteBMpOV4WvafFwnOaCLh1N80dXYTTRuOhEOtdXTHCepM/zASS314iqU+rLQMJ9xbjzn9\ntQhZ+rEH/dKivJ6/sBnqSUC9CEwxs8kEwXQV8MH0BcxsDvAT4CJ33542vQyod/cmMxsJnEHQgEKk\nU42xBNW7G9i0q54NO/ezcVcDG3ftZ+OuejbtaqAhluh2G2ZQGA23voGHW9/E2vrh1H+6YLjtMkc0\nFCISClEQDdaNhNPeUFPba3lzj4Tbv8lGw2nBELbgclc0REE0HHSRtOGW6ZEw0UjbO07LG3Iw7GnD\nbcfW8uYV7uk7lUg/1W1AuXvczG4CHgPCwL3u/qqZ3QFUufsS4LvAEODXFnzi2OjulwHTgJ+YWZKg\nQca3OrT+k0EgmXT2NMaorY+xu76Z2oYYtfXNqfEYW2ob2LCrnk276tm6p7HddwdFeWEmDC9i0ohi\nzpxSTllRlCH5EYYUpPr5EYYUBP2hBRGK8yMURcOE9OYt0u+Z9/SbxCOksrLSq6qq+roM6YFE0tmx\nr4nNtQ28XdvAltrG1uHte5uCEGqIUdcQO+gX1qOG5jNxRBHjhxcxcXgxE0YUMmF4MROGFzFySB5m\nChuRgcTMlrt7t19kZbWRhAwsDc0JNtc2UL27ns21DWzeHYTP23WNvF3bwNa6RuIdvnwpzgtzVGkh\no4cVUFFWSFlRHmVFUUpS/dKiKKVFeZQWRikrymNYYVSXqkSkUwqoQSyRdNbv2MeGnfVU725oC6PU\n8I59ze2WD4eMMcMKGFdayMkTyziqtDDoSgpah4cVRHTGIyJZoYAaJNydjbvqWVldx6pNtazaXMer\nm+vY39zW6CAvEqKitJBxZYVMGzuMirJgeFxpERVlwVmRznZE5EhRQA1QW+saWVldy6rqWlZV17Gq\nuo66hhgQBNEJRw3j/SdXMLOilKPLixlXVsjI4nw1LhCRnKGAGkD2N8X5zUvVLHpuA29s3wcEl+WO\nHz2Ui08cw8yKUmZWlHD8mKFEw/3sRvbukGiGRAw8AclU5+n9OCSTwbC3/OjUUr92TQVvy7Clj9N+\nGp3MaxlPxiERh2QsVU886CdjQW2JWNuwp/3w1Z3ghy/pfdqGPdnW0TLeYbong1pCEQiFg87CaeMR\nsFDbeLwRmushVg/N+4Ous+F4Y/t27J2+Bt1Max3N0gecg+2jVYeWN521xGl5vdOHO7727fZpXfTT\n5rf+zdK21/p36zB8KLUedP8d/h2n/1vq6tjabzxtH2nj6dO63F6HaZPOhAv/48D6DwMF1ACwfsd+\nFj33FourqtnbFGdmRQlffvd0Zo0v5YSjhlEQDfduw/Fm2F8D+7YF/YbdwRtg+ptgKJJ6k+zwholD\nrCHo4o3Bm2Es1Y83ps1raBtu7eo7Ga7nwP90ckgsBNFiyCuCvOK24UhBMK/dGxG0e727fPNvN6Ft\nWkatgw+2X6fTN9e2CR1G097crcO0dsOdfHhoGU8m28/HUv8PUuuGQmCRA6e3C5Tuau24/676XRxb\ny/YOODYO/Ht2FpzuB99e+rTCUo4UBVQ/lUw6f32jhoXPvsVTr9UQDRuXzBjLR06fxJzxpe0bKiST\n0LwPmvZC0562fuOeoL9/B+zbDvu3w76aVH87NNYevgMI50G0ECKFQT9alOoXwpDRbcPp8yIFEI6m\nBWLLGUOHoGwJUeCA/9gH+2TYOkzn4+7BfsLRoAtFOwznQTgS9Ftq6/Q/PAe+kVhL3dZWf2tnbdvC\nO5w9xlPj8eCTe/p4pCAIoGhxEEiR/Oyd4YgcAQqofmZPY4zFVdXc9/wG1u/YT/nQfD513hQ+eOoE\nRg0tgPpd8MdPwcYX2gdSd2cf+cOguByGjILyqTD5LCgeBUPKg8AoHgVFw4Nl270ppl1uSx+HVLAU\npIVQYVvQhHp5Vicig4YCKkfEEsngDtKx4J5yHe8iXd+UYNXmWn7z0mbqmxOcNKGUT101m4tPHEte\nJBR8ul/1MPzpNmisg2PPDwIlf2gQPvlDoSDVzy9pP140IggOEZEcooA6jFrutLBtTyNb6xrZtreJ\nbXWNbN3TyLZUV7O3iX1NcWKJ7q/Z50VCvGfmUVx7+qT2t7vf/Rb88dPw5hMw7mR4zw9hzImH78BE\nRI4ABVQvuDu19TG27W1k254ggLbvCYa3pg3X7Gsi0eFOC+GQUT4kn9ElBUwaUczcycMZWhClKBqm\nKD+4i3RhXpjilrtIp00bXpxHUV7anywRhxfugqXfDL6juPg7cMr1unwmIgOCAqoLdfUx1u/cz1s7\n9rN+x37e2rmfzbsbWkOpsweHlRZFGT20gFHD8jlu9FBGDytgdEkBY4YVMHpYPmOGFTBiSH52fuy6\nZSUsuQW2rIDjLoZLvwclepCxiAwcgzqgEklnzZY9rNsRBNFbO/a3htLu+ljrcmZwVEkhk8qinDku\nzLgp+YwtTDAmv5mRec0Mj8QoCTUQTdSnGiakGiVYGPZHoCEM29OaYre2OEu1CCupgLJJUDohaGl1\nMM318NQ34bn/C8Uj4cqFMH2+WmeJyIAzKANq9dt7+O1Lm3hsxXqa9+1ihO1luO3hmKIGrihqZMKI\n/YwZvY/h1DE0WUd+0y5C9Tthy96e7SCcHwRRS2s37/4ZRgGDYeOCsOqs27oK/ngr1G6Akz4C538N\nCst68QqIiOS+gRdQ9btg+S/amlenfuvTvL+WPXW7iNfXclRiP5+ngS9aEgrS1o0DewjObIrLoWhk\ncJYy6phgvLAsaPWWNwTyh6SGh6YNp/rhaPuaWn5dnh5YLXc9iDdCXXXQ0GH3+lT/raDBw94tBx7f\niClw7SMw6YzD9AKKiOSGgRdQTXvhia9BKILnD2O/FVMTy2dbU5S9PpRIcQVjR41i4lFjiAwpC5pa\nF5e3D6SCkuxeMmv5IWZXjRdKxsGEUw+cHmuA2o1toWUhmPPh4LdFIiID3IALqPjQcTz3gVdYvLKG\nP6/eTkMswfjhhVwxt4Ir5oxj8sjivi6x56KFUH580ImIDDIDLqDW7qjnw4tWMawgwhUnjeO9c8Zx\n8sQyPaNIRKSfGXABNXXMMBZ+dC6nHT2c/Ih+DyQi0l/16JkLZnaRmb1mZmvN7LZO5ueb2UOp+S+Y\n2aS0eV9ITX/NzC7MXuldO/u4coWTiEg/121AmVkYuBO4GJgOXG1m0zss9jFgt7sfC3wf+HZq3enA\nVcAJwEXA/01tT0RE5KB6cgY1F1jr7uvcvRl4EJjfYZn5wMLU8GJgngVf+swHHnT3JndfD6xNbU9E\nROSgehJQ44BNaePVqWmdLuPucaAOGNHDdUVERA6QE8/9NrMbzazKzKpqamr6uhwREckBPWnFtxkY\nnzZekZrW2TLVZhYBSoCdPVwXd78buBvAzGrMbENPD+AgRgI7srCdXDIQjwl0XP3JQDwmGJjHlcvH\nNLEnC/UkoF4EppjZZIJwuQr4YIdllgAfAZ4D3g886e5uZkuAX5rZfwFHAVOAvx9sZ+5e3pPCu2Nm\nVe5emY1t5YqBeEyg4+pPBuIxwcA8roFwTN0GlLvHzewm4DEgDNzr7q+a2R1AlbsvAX4G3Gdma4Fd\nBCFGarmHgdUEd7r7hHuP75wqIiKDWI9+qOvujwCPdJj25bThRuDKLtb9D+A/MqhRREQGoZxoJHGY\n3N3XBRwGA/GYQMfVnwzEY4KBeVz9/pjM3btfSkRE5AgbyGdQIiLSjymgREQkJymgREQkJymgREQk\nJymgREQkJymgREQkJymgREQkJymgREQkJymgREQkJymgREQkJymgREQkJymgREQkJymgREQkJymg\nREQkJ/XogYVH0siRI33SpEl9XYaIiBwmy5cv3+Hu5d0tl3MBNWnSJKqqqvq6DBEROUzMbENPltMl\nPhERyUkKKBERyUkDLqC21jXyhd+sYu32fX1dioiIZCDnvoPKVCRsLF5eTUE0zFfec0JflyMiA1As\nFqO6uprGxsa+LiWnFRQUUFFRQTQa7dX6Ay6gRg7J56ITx/K/y6v53IVTKcwL93VJIjLAVFdXM3To\nUCZNmoSZ9XU5Ocnd2blzJ9XV1UyePLlX2xhwl/gArjl1Ansa4/xh5dt9XYqIDECNjY2MGDFC4XQQ\nZsaIESMyOssckAE1d/Jwjhs9hPtf6FFLRhGRQ6Zw6l6mr9GADCgz40OnTmRVdR2rqmv7uhwREemF\nARlQAFecNI7CaJj7n9dZlIgMPEOGDOnrEg67ARtQwwqiXD7nKJasfJu6+lhflyMiIoco41Z8ZhYG\nqoDN7v7uDvOuBb4LbE5N+pG735PpPnvqQ6dO5Fd/38T/vlTNR9/Zu1YkIiIH87U/vMrqt/dkdZvT\njxrW45/JuDuf+9znePTRRzEzvvSlL7FgwQK2bNnCggUL2LNnD/F4nLvuuovTTz+dj33sY1RVVWFm\nfPSjH+XWW2/Nau3ZlI1m5p8E1gDDupj/kLvflIX9HLITx5Uwe3wpD7ywgevOUHNQERl4fvOb37Bi\nxQpWrlzJjh07OOWUUzjrrLP45S9/yYUXXsgXv/hFEokE9fX1rFixgs2bN/PKK68AUFub29/RZxRQ\nZlYBXAr8B/DprFSUZdecNpHP/Holz63byenHjOzrckRkgOnrGwI8/fTTXH311YTDYUaPHs3ZZ5/N\niy++yCmnnMJHP/pRYrEYl19+ObNnz+boo49m3bp13HzzzVx66aVccMEFfVp7dzL9Duq/gc8ByYMs\n8z4zW2Vmi81sfGcLmNmNZlZlZlU1NTUZltTeu2eOpaQwygPPb8zqdkVEctlZZ53FsmXLGDduHNde\ney2LFi2irKyMlStXcs455/DjH/+Y66+/vq/LPKheB5SZvRvY7u7LD7LYH4BJ7j4TeBxY2NlC7n63\nu1e6e2V5ebePCDkkBdEwV55cwWOvbmX7Ht2WREQGljPPPJOHHnqIRCJBTU0Ny5YtY+7cuWzYsIHR\no0dzww03cP311/PSSy+xY8cOkskk73vf+/jGN77BSy+91NflH1Qml/jOAC4zs0uAAmCYmd3v7te0\nLODuO9OWvwf4Tgb767UPnTaRe55ez0MvbuLmeVP6ogQRkcPiiiuu4LnnnmPWrFmYGd/5zncYM2YM\nCxcu5Lvf/S7RaJQhQ4awaNEiNm/ezHXXXUcyGVz0+s///M8+rv7gzN0z34jZOcBnOmnFN9bdt6SG\nrwA+7+6nHWxblZWVfjgeWHjNPS+wrmYff/v8uYRDaiwhIr23Zs0apk2b1tdl9AudvVZmttzdK7tb\nN+u/gzKzO8zsstToLWb2qpmtBG4Brs32/nrqmtMm8HZdI0/+c3tflSAiIocgK3czd/engKdSw19O\nm/4F4AvZ2Eemzps2mtHD8rn/+Q2cP310X5cjIiLdGLB3kugoEg5x1SkTWPZGDRt31vd1OSIi0o1B\nE1AAV8+dQMiMB/6u+/OJiOS6QRVQY0oKOG/aKH5dVU1TPNHX5YiIyEEMzIBa8Uto7vwy3jWnTWTX\n/mYefXnrES5KREQOxcALqLdXwO8+DnedDuuXHTD7jGNGMmlEkR7DISKS4wZeQB01G/5lSTC88D3w\n+5ugYXfr7FAoeJhh1Ybd/HNrdu9ALCKSiw727Ki33nqLE0888QhW03NZaWaec44+Gz7+LPz1W/Ds\nj+CNP8Ml34Xp8wF4/8kVfPfPr3H/8xv4xuUz+rhYEenXHr0Ntr6c3W2OmQEXfyu72+yHBt4ZVIu8\nIjj/DrjhSRgyCh7+F3jwQ7BnC2XFebx75lh++9Jm9jXF+7rSnmnaCzWv93UVIpIDbrvtNu68887W\n8a9+9at84xvfYN68eZx00knMmDGD3//+94e83cbGRq677jpmzJjBnDlzWLp0KQCvvvoqc+fOZfbs\n2cycOZM33niD/fv3c+mllzJr1ixOPPFEHnrooawdXyt3z6nu5JNP9qyLN7v/7b/cvz7K/Zvj3at+\n7svf2ukTP/9Hv++5t7K/v2zbvcH9f05x/9oI973b+7oakUFv9erVfbr/l156yc8666zW8WnTpvnG\njRu9rq7O3d1ramr8mGOO8WQy6e7uxcXFXW5r/fr1fsIJJ7i7+/e+9z2/7rrr3N19zZo1Pn78eG9o\naPCbbrrJ77//fnd3b2pq8vr6el+8eLFff/31rdupra3tdPudvVZAlfcgDwbuGVS6cBTeeWtw2W/s\nTPjDJ5mz9MOcN3of9z+/gWQy8/sRHjZbVsE950FdNSRj8M8/9HVFItLH5syZw/bt23n77bdZuXIl\nZWVljBkzhttvv52ZM2dy3nnnsXnzZrZt23ZI23366ae55prgft9Tp05l4sSJvP7667zjHe/gm9/8\nJt/+9rfZsGEDhYWFzJgxg8cff5zPf/7z/O1vf6OkpCTrxzk4AqrFiGOCBhTv+QG25WV+svdmLqhZ\nyDd/9iv21zf0dXUHWvsE/PxiCEXhhidg+DGw+tBP20Vk4LnyyitZvHgxDz30EAsWLOCBBx6gpqaG\n5cuXs2LFCkaPHk1jY3YeMfRKLh6bAAAgAElEQVTBD36QJUuWUFhYyCWXXMKTTz7Jcccdx0svvcSM\nGTP40pe+xB133JGVfaUbmI0kDiYUgpOvhSkXEnrkM3z6n4th82KavvNJmsbMIH/iKTDu5KAbfjT0\n1WPiV/wSltwM5VPhQ7+GYUcFjTye+QHs3wnFI/qmLhHJCQsWLOCGG25gx44d/PWvf+Xhhx9m1KhR\nRKNRli5dyoYNh/5TmjPPPJMHHniAc889l9dff52NGzdy/PHHs27dOo4++mhuueUWNm7cyKpVq5g6\ndSrDhw/nmmuuobS0lHvuuSfrxzj4AqrFsLHYVQ/A7g28WrWUF59+nJlb32TWjoWEX/hxsExBCRx1\nUiqwToKJp0Nh2eGtyx3+9j148hsw+WxYcD8UDAvmnXA5PP1f8M8/wskfObx1iEhOO+GEE9i7dy/j\nxo1j7NixfOhDH+I973kPM2bMoLKykqlTpx7yNv/t3/6Nj3/848yYMYNIJMIvfvEL8vPzefjhh7nv\nvvuIRqOtlxJffPFFPvvZzxIKhYhGo9x1111ZP8asPA8qmw7X86C6s3b7Xj62sIrttfv54XlFnF9S\nDZuXB9221eAJKCiF9/w3nHDF4SkiEYdHPgPLfw4zF8BlP4JIXtt8d/jh7OBS34d/c3hqEJFu6XlQ\nPZdTz4Pqr44dNZTf/dsZzJ44khsea+Bb2+aSuPS/4f88DV+ohmsfCb7D+vW18Nv/A4112S2geT88\ndE0QTu+8Fa74SftwguBy4/T5sP6vUL8ru/sXEckxCqg0ZcV5LPrYXD546gR+/Nc3+df7qoLfSeUV\nwaQz4KOPwdm3waqH4a53woZns7PjfTXBXS/eeAwu+R6c99Wuv/uafjkk4/DaI9nZt4gMCi+//DKz\nZ89u15166ql9XdZB6RJfJ9ydhc++xR1/XM1xo4fy03+pZPzworYFNv0dfnMj7H4rONs55wsHnu30\n1M434f73wd6t8P6fwdRLuysO/nsmjEo1nhCRI27NmjVMnToV66tGVP2Eu/PPf/5Tl/iyycy49ozJ\n/OK6uWyubeDyO5/hxbfSLqmNnxtc+jvpw0GjhZ+dd2h3eUjEghvZ/un24DdOjXXwkT90H05BcTD9\nMnhzKTTUHvrBiUjGCgoK2LlzJ7n2AT+XuDs7d+6koKCg19vQGVQ33qzZx/ULq6jeXc+X3z2dq+dO\nIBJOy/U1fwyag8ca4IKvwynXd355bv9OWPs4vP6n4PdNTXsgnB/cN/DC/4SRx/a8qOoquGceXP5j\nmH115gcpIockFotRXV2dtd8ZDVQFBQVUVFQQjUbbTe/pGZQCqgdq65u5+Vf/4G9v7ODo8mI+ff5x\nXHLiWEKhVBDt3Qq//wSs/Qscez7MvzO4/9/2NUEgvf4YVP8dPAlDRsNxF8JxFwXNyPO7vstwl9zh\n+ycGN5T84IPZPVgRkcNMAZVl7s5jr27j//vza7yxfR/Txw7jMxcex7uOHxVch3aHF++BP38JokVB\n8NRuDFYeOzsIpOMvgjGzgh8LZ+pPt8OLP4XPvtn2OykRkX5AAXWYJJLOkpWb+f7jb7BxVz0nTyzj\nMxcczzuOSd3ZoeY1eOx2COcFoTTlAhg2NvuFbHwB7r0A3vtTmPmB7G9fROQwUUAdZrFEkoerNvE/\nT6xl655G3nnsSD5z4fHMHl96ZApIJuH7JwR3uLjqgSOzTxGRLFArvsMsGg7xoVMn8tRnz+FLl05j\n9ZY9XH7nM9ywqOrIPKk3FApa873xePCsKBGRASbjgDKzsJn9w8z+2Mm8fDN7yMzWmtkLZjYp0/3l\nmoJomOvPPJpln3sX/37+cTz/5k4u/sHfuHFRFU+/sePwNkOdPh8STUEjDBGRASYbZ1CfBNZ0Me9j\nwG53Pxb4PvDtLOwvJw3Jj3DzvCn87fPv4hPnHEvVht1c87MXmPdff+Xnz6xnT2Ms+zsdf2rQKlCP\n4BCRASijgDKzCuBSoKv7rM8HFqaGFwPzbID/9Lq0KI/PXHg8z952Lv/1gVkMK4jytT+s5tT/eIIv\n/OZl1mzJ4uW/UBimpS7zNe/P3nZFRHJApmdQ/w18Dkh2MX8csAnA3eNAHXDAg4zM7EYzqzKzqpqa\nmgxLyg0F0TDvPamC333iDP5w0zt5z6yx/Oalai7+wd+48sfPsmTl2zTHu3rZDsH0+RBvgDf+nPm2\nRERySK8DyszeDWx39+WZFuHud7t7pbtXlpeXZ7q5nDOjooTvvH8WL9w+jy9eMo1te5q45Vf/4PRv\nPcn3HnuNv6/fRUNzoncbn3g6FJfrMp+IDDiZPLDwDOAyM7sEKACGmdn97n5N2jKbgfFAtZlFgBJg\nZwb77NdKi/K44ayj+dg7J/PXN2q477kN3PnUWn60dC2RkDFt7DDmTCjlpAllzJlQyoThRd3fjDIU\nhmnvgZUPQXN9cOd1EZEBICu/gzKzc4DPuPu7O0z/BDDD3f+PmV0FvNfdD/qr0v7yO6hs2bmviRWb\nanlp427+sbGWlZtq2Z86mxpRnMecCaXMmVDGnPGlzJ5QSlFeJ58p1j0Fi+bDB+4Lmp6LiOSwnv4O\nKuuPfDezO4Aqd18C/Ay4z8zWAruAq7K9v/5uxJB85k0bzbxpo4HgThWvb9vLPza2hNZu/rJmOwCj\nh+Xz50+dTUlR+xsvMvGdUDQiuMyngBKRAUJ3kugH6upjLHujhpt/9Q8+OW8Kt55/3IELLbkFXvnf\n4N580d7f3l5E5HDTnSQGkJKiKO+ZdRQXTB/Nvc+sp66hk99UTZ8PzfvgzSeOfIEiIoeBAqofuWXe\nFPY2xln47FsHzpx8FhSWqTWfiAwYCqh+5MRxJZw3bRQ/e3o9ezvemSIcDZ7I+9qjEG/qmwJFRLJI\nAdXP3DJvCnUNMRY9t+HAmdMvD57Uu+6pI16XiEi2KaD6mZkVpbzr+HLu+ds69jfF28+cfDYUlMCr\nv+ub4kREskgB1Q/dMm8Ku+tj3Pd8h7OoSB4cfym89v8g3tw3xYmIZIkCqh+aM6GMs44r56fL1lHf\n3OEsavp8aKyD9cv6pjgRkSxRQPVTn5x3LDv3N/PA8xvbzzjmXZA/LHjs/COfhWf/J2jZ9/Y/oH4X\n5Njv3kREupL1O0nIkXHyxOGccewIfrJsHdecNpHCvHAwI5IP77od/vEArHwwaDSRLm8IlE5o342d\nDRWVEC088gciItIFBVQ/9sl5x/GBnzzHr/6+kY++c3LbjNM+HnQADbVQu7HzbsOzbQEWisK4k4K7\no088A8bPDRpciIj0EQVUPzZ38nBOO3o4P/7rm3zw1AkURMMHLlRYGnRjZ3a+kfpdUP0ibHgmCKxn\n/wee/j5YCMbMgAmnp0LrdCge2bZevDn4rqu1q20/nohB/tCgKxiWGh6W6lLTIgUwsJ9fKSIZ0L34\n+rln39zBB3/6Al+77AQ+cvqkzDfYvB+qq4Kw2vBMEF7xxmBeyQRIxoKzsnhD5vsKRYLAKi6HYUdB\nyTgYVpHqH9U2nD80832JSM7os7uZy5H1jqNHMHfScO566k2umjue/EgnZ1GHIq8Yjj476CA4U9qy\nIgirra8Ez5sqKEl1pWnDJe2nh6PQtDe4hNi4p224s2n7tkHdZti+Jhimw4em/JJUYB0F4bxOiu7i\nQ5aFIRRK9SPBs7NapoUiqeG0ea3TIu2ntQ6nj0c7jEcgHGk/biHAgr7RYdzaj4ejaduJpvWjabUb\nJJPQvDf1+u3p0K9rG2/aG7xW+UOD7x1bzmbTu7whwQeEvOJg++1qO8xntu6pBjud/e3S9t1Sh860\nByUFVD9nZtwybwrX/OwFHq6q5sOnTczuDiJ5wfdR4+ce+rpFw4PuUMSbYe8W2LM5CK09m9uG926B\nZPzAdTp783IHT0IyAZ4I+sl4h2nx1PT08dQyuSgUTR1/N1c9QpEggBLxIMx6q12gpoIrPTw61tHu\nakxaAHXsZ03H8OoszDr5t9Fp2HURgIeybLfr20HmHeJ2u91XFw71illn2zzuQnjfPYe2nV5SQA0A\nZxw7gpMnlnHX0rUsqBxPXqQf/3ogkgdlE4Our7i3BVpr13G847RY+/FEjNY35ZawJNU/YDwZhEnL\ndhKx9ttJ74ciqe/0hqX1S9qPRwvb3liSSYjtT52t7oWmfW1nWM372vrpdXVVb8u0zs5w2iZ0mGc9\n7Ke99m0jPZjWxfgB63Hg/IMud6jLdrN+d+tl/FVLF7VmFLBd1DRqWk+LypgCagBoOYv6yL1/539f\nqubquRP6uqT+zSy4XBceAP89QqG2S3oi/Uw//qgt6c6aMpJZ40u5c+laYokcvUQlInIIFFADhJnx\nqXlTqN7dwG9f2tzX5YiIZEwBNYCcc3w5M8aV8KOla4nrLEpE+jkF1ADS8l3Uxl31/G7F231djohI\nRhRQA8x500YxY1wJX/79K/z51a19XY6ISK8poAYYM+Oej1QyZdQQ/vX+5dz11Jvk2t1CRER6QgE1\nAI0eVsBD//oOLp0xlm//6Z/8+69X0hRP9HVZIiKHpNcBZWYFZvZ3M1tpZq+a2dc6WeZaM6sxsxWp\n7vrMypWeKoiG+Z+r53Drecfxm5c288GfvsCOfU19XZaISI9lcgbVBJzr7rOA2cBFZnZaJ8s95O6z\nU92RuT+GAMHlvk+eN4U7P3gSr75dx/wfPcOaLXu6X1FEJAf0OqA8sC81Gk11+rIjB106cyy//tfT\nSSSd9931rBpPiEi/kNF3UGYWNrMVwHbgcXd/oZPF3mdmq8xssZmNz2R/0nszKkr4/U1nqPGEiPQb\nGQWUuyfcfTZQAcw1sxM7LPIHYJK7zwQeBxZ2th0zu9HMqsysqqamJpOS5CDUeEJE+pOsPbDQzL4M\n1Lv797qYHwZ2uftBnyOuBxYefu7OD59Yy/f/8jonTSjlh1fPoaKsqK/LEpFBoqcPLMykFV+5mZWm\nhguB84F/dlhmbNroZcCa3u5Psie98cTqLXt41/ee4gu/WcWmXfV9XZqISKtMnicwFliYOjMKAQ+7\n+x/N7A6gyt2XALeY2WVAHNgFXJtpwZI9l84cy+wJpfz4qTd56MVNPFxVzRVzxvGJdx3L5JHFfV2e\niAxyWbvEly26xNc3ttY18pNlb/LLFzYSSyS5bNZR3HTusRw7Ss8REpHs6uklPgWUtLN9byP3/G09\n9z23gcZ4gktnjOXmc6dw/BgFlYhkhwJKMrJzXxM/e3o9C599i/3NCS48YTQ3nzuFE8cdtI2LiEi3\nFFCSFbX1zdz7zFv8/Jn17G2Mc+aUkdxw5tGcOWUkZtbX5YlIP6SAkqyqa4hx//MbWPjsW2zf28TU\nMUO5/syjuWzWUeRFdM9hEek5BZQcFk3xBH9YuYWfLlvHa9v2MnpYPteePpkPnjqBksJoX5cnIv2A\nAkoOK3dn2Rs7+OmydTy9dgfFeWE+cMp4PnrGZMYP149+RaRrCig5Yl59u46f/W09S1a+TdKdi2eM\n5brTJ3HShDJCIX1PJSLtKaDkiNtS18AvnnmLX76wkb1NcUYNzee86aM5f/poTj9mBPmRcF+XKCI5\nQAElfWZvY4wn1mznz6u38tfXatjfnKA4L8w5x4/i/Omjedfxoygp0vdVIoOVAkpyQmMswXNv7uTP\nq7fxlzXbqNnbRCRknHr0cM6fNprzTxjDuNLCvi5TRI4gBZTknGTSWVFdy+Ort/HnV7fyZs1+AE4/\nZgS3nn8cp0wa3scVisiRoICSnLeuZh+PvrKVnz/zFjv2NXHmlJHcev5xnDShrK9LE5HDSAEl/UZD\nc4L7n9/AXX99k137m3nX8eXcev5xzKwo7evSROQwUEBJv7O/Kc6i5zbwk2VvUlsf47xpo/nUebr/\nn8hAo4CSfmtvY4yFz77F3cvWsacxzkUnjOFT509h6phhfV2aiGSBAkr6vbqGGPc+vZ57n17P3qY4\nl84YyzuOGcG4skLGlRZyVGkhQ/IzeeamiPQFBZQMGLX1zfzs6fX8/Jm32NcUbzevpDDKUaVBYI0r\nLWBcWRBcY4YVUJgXpjAapiDa1s+PhHR3C5E+poCSASeRdGr2NrG5toHNtQ28XdvA5t2pfqrb2xjv\ndjv5kVBaaIUYUhBheHE+I4rzGFGcx/AhqX5xPiNah/MYkh/RI0ZEsqCnAaXrI9JvhEPGmJICxpQU\ncPLEzpui72mM8XZtA9v2NNHQnKAxFnQNsQSNsSQNsQRNreMJGmJJ9jbG2LW/mTe372PX/mYaYolO\nt50XCVFSGGVoQYShBVGGFUQYWhBhWEHbtPR5pUV5lBZFKS2MMqwwSkFUt3oSORQKKBlQhhVEGTYm\nytQxvd9GQ3OCnfub2LmvmV37m9m5v5ld+5vYub+ZPQ0x9jTG2dsYZ09DEIZ7U+NdBVuLgmiI0sIg\ntEoKo6nwymNYYaT18mNBNEx+NExBarggdZaXHwn6QwuijBiSx1CdzckgoIAS6aAwL0xFXhEVZYf2\n2JBYIpkKqxh7GuLUNcSobWimtj5GXUPQ1dYH47UNMd7aUU9tQy17GuI0xhMcytX2vHAouPw4JI8R\nqUuRI4fkt16OHDkkn6K81Pdu0RAFkaDfEnT5kTBhfRcnOU4BJZIl0XCI4amAOFTuTnMiSWMsSVPq\ncmRTPOg3xlsuVQaXI3fua2ZH6gxv577gzG7t9n3s2NdEUzzZ431GQtZ6ljZ6WD5jU5dPx5YUthtu\naXDSF1pel+Z4kqZ4kkTSKc6PUJwX1hnkIKCAEskBZkZ+JBw8kqSXTyZ2d+qbE60B1tCcoCmeoCkV\nck2xJI2xBE2pN/uW4frmONv2NFG9u4GqDbuprY8dsO3SoihjhhVQWhQlnnBiSScWTxJPJoknghCJ\nJ5xYIpnqHLPge8NIyAiHQkTDljZuREIhIqlpsYS31toUT9Icb6uzMyGDIflt3/u1fQ+Y+g6wMEJh\nNEw4FCIcIuin6mmZFjIjEjZCFnTB3wEMoyX7LDUtGArEEslUbW31NrXUG2sbjieShMxS+zRCqWNv\nmRZJTQun6oiGg9co6HccDsYj4RDJpHe6v7aura5E6u8TT3rr3yoYduKJZGs/4cG/n6Q7ySQ4aeMO\nSXc8tcwZx47kP66Y0at/o4dKASUyQJhZcHaRH2HCiN4/1bihOcHWPY1sqWtga10jW+oaW/t7GmLk\nRUIUt7xhhkJEIyGioeDNNP2NFiCedBJJJ5bw4M0yNd76xpgajoZD5EdC5EVCqaAOBZckwyHyU9/P\n5UdCmBn7m+Ktl1L3NsZT3wnG2FLXyOvbY63fCSaSR7aFcl7qGFoupUbCRiLpJFPHmPS2Y08mnURq\nPJEMQiAbzEi9VmHyUn+XcOrv1PLBIBoOtQZkJGwU5UVSQRmEtlnwbymUGu84bfLI4uwU2wO9Digz\nKwCWAfmp7Sx29690WCYfWAScDOwEFrj7W72uVkQOu8K8MJNHFh/RN6Jsa7k0mEzSLggSqaBoDYlU\nYEDqDAFavwv0lmlp49AWAK39aIi8cGa/r0smnVgyOPOMJ5I0p85CW85Sm+PB2Wk8mSQcCrUGdnp4\n50fCRMM2oC59ZnIG1QSc6+77zCwKPG1mj7r782nLfAzY7e7HmtlVwLeBBRnsU0SkWy2XTPuLUMjI\nD4XRjVHaC/V2RQ/sS41GU13HE9X5wMLU8GJgng2keBcRkcOm1wEFYGZhM1sBbAced/cXOiwyDtgE\n4O5xoA4Y0cl2bjSzKjOrqqmpyaQkEREZIDIKKHdPuPtsoAKYa2Yn9nI7d7t7pbtXlpeXZ1KSiIgM\nEFm54unutWa2FLgIeCVt1mZgPFBtZhGghKCxRJeWL1++w8w2ZKGskcCOLGwnlwzEYwIdV38yEI8J\nBuZx5fIxTezJQpm04isHYqlwKgTOJ2gEkW4J8BHgOeD9wJPezd1p3T0rp1BmVtWTmxH2JwPxmEDH\n1Z8MxGOCgXlcA+GYMjmDGgssNLMwwaXCh939j2Z2B1Dl7kuAnwH3mdlaYBdwVcYVi4jIoNDrgHL3\nVcCcTqZ/OW24Ebiyt/sQEZHBK6NGEjnu7r4u4DAYiMcEOq7+ZCAeEwzM4+r3x5RzDywUERGBgX0G\nJSIi/ZgCSkREcpICSkREcpICSkREcpICSkREcpICSkREcpICSkREcpICSkREcpICSkREcpICSkRE\ncpICSkREclKmj3y/18y2m9krXcw3M/uhma01s1VmdlIm+xMRkcEj0zOoXxA8RbcrFwNTUt2NwF0Z\n7k9ERAaJjALK3ZcRPIiwK/OBRR54Hig1s7GZ7FNERAaHTJ6o2xPjgE1p49WpaVvSFzKzGwnOsCgu\nLj556tSph7ksERHpK8uXL9/h7uXdLXe4A6pH3P1uUg/Xqqys9Kqqqj6uSEREDhcz29CT5Q53K77N\nwPi08YrUNBERkYM63AG1BPiXVGu+04A6d9/S3UoiIiIZXeIzs18B5wAjzawa+AoQBXD3HwOPAJcA\na4F64LpM9iciIoNHRgHl7ld3M9+BT2SyDxERGZx0JwkREclJCigREclJCigREclJCigREclJCigR\nEclJCigREclJCigREclJCigREclJCigREclJCigREclJCigREclJCigREclJCigREclJCigREclJ\nGQWUmV1kZq+Z2Vozu62T+RPMbKmZ/cPMVpnZJZnsT0REBo9eB5SZhYE7gYuB6cDVZja9w2JfAh52\n9znAVcD/7e3+RERkcMnkDGousNbd17l7M/AgML/DMg4MSw2XAG9nsD8RERlEMgmoccCmtPHq1LR0\nXwWuST0O/hHg5s42ZGY3mlmVmVXV1NRkUJKIiAwUh7uRxNXAL9y9ArgEuM/MDtinu9/t7pXuXlle\nXn6YSxIRkf4gk4DaDIxPG69ITUv3MeBhAHd/DigARmawTxERGSQyCagXgSlmNtnM8ggaQSzpsMxG\nYB6AmU0jCChdwxMRkW71OqDcPQ7cBDwGrCForfeqmd1hZpelFvt34AYzWwn8CrjW3T3TokVEZOCL\nZLKyuz9C0PghfdqX04ZXA2dksg8RERmcdCcJERHJSQooERHJSQooERHJSQooERHJSQooERHJSQoo\nERHJSQooERHJSQooERHJSQooERHJSQooERHJSQooERHJSQooERHJSQooERHJSQooERHJSRkFlJld\nZGavmdlaM7uti2U+YGarzexVM/tlJvsTEZHBo9fPgzKzMHAncD5QDbxoZktSz4BqWWYK8AXgDHff\nbWajMi1YREQGh0zOoOYCa919nbs3Aw8C8zsscwNwp7vvBnD37RnsT0REBpFMAmocsCltvDo1Ld1x\nwHFm9oyZPW9mF3W2ITO70cyqzKyqpqYmg5JERGSgONyNJCLAFOAc4Grgp2ZW2nEhd7/b3SvdvbK8\nvPwwlyQiIv1BJgG1GRifNl6RmpauGlji7jF3Xw+8ThBYIiIiB5VJQL0ITDGzyWaWB1wFLOmwzO8I\nzp4ws5EEl/zWZbBPEREZJHodUO4eB24CHgPWAA+7+6tmdoeZXZZa7DFgp5mtBpYCn3X3nZkWLSIi\nA5+5e1/X0E5lZaVXVVX1dRkiInKYmNlyd6/sbjndSUJERHKSAkpERHKSAkpERHKSAkpERHKSAkpE\nRHKSAkpERHKSAkpERHKSAkpERHKSAkpERHKSAkpERHKSAkpERHKSAkpERHKSAkpERHKSAkpERHJS\nRgFlZheZ2WtmttbMbjvIcu8zMzezbm+vLiIiAhkElJmFgTuBi4HpwNVmNr2T5YYCnwRe6O2+RERk\n8MnkDGousNbd17l7M/AgML+T5b4OfBtozGBfIiIyyGQSUOOATWnj1alprczsJGC8u/+/g23IzG40\nsyozq6qpqcmgJBERGSgOWyMJMwsB/wX8e3fLuvvd7l7p7pXl5eWHqyQREelHMgmozcD4tPGK1LQW\nQ4ETgafM7C3gNGCJGkqIiEhPZBJQLwJTzGyymeUBVwFLWma6e527j3T3Se4+CXgeuMzdqzKqWERE\nBoVeB5S7x4GbgMeANcDD7v6qmd1hZpdlq0ARERmcIpms7O6PAI90mPblLpY9J5N9iYjI4KI7SYiI\nSE5SQImISE5SQImISE5SQImISE5SQImISE5SQImISE5SQImISE5SQImISE5SQImISE5SQImISE5S\nQImISE5SQImISE5SQImISE5SQImISE7KKKDM7CIze83M1prZbZ3M/7SZrTazVWb2hJlNzGR/IiIy\nePQ6oMwsDNwJXAxMB642s+kdFvsHUOnuM4HFwHd6uz8RERlcMjmDmgusdfd17t4MPAjMT1/A3Ze6\ne31q9HmgIoP9iYjIIJJJQI0DNqWNV6emdeVjwKOdzTCzG82sysyqampqMihJREQGiiPSSMLMrgEq\nge92Nt/d73b3SnevLC8vPxIliYhIjotksO5mYHzaeEVqWjtmdh7wReBsd2/KYH8iIjKIZHIG9SIw\nxcwmm1kecBWwJH0BM5sD/AS4zN23Z7AvEREZZHodUO4eB24CHgPWAA+7+6tmdoeZXZZa7LvAEODX\nZrbCzJZ0sTkREZF2MrnEh7s/AjzSYdqX04bPy2T7IiIyeOlOEiIikpMUUCIikpMUUCIikpMUUCIi\nkpMUUCIikpMUUCIikpMUUCIikpMUUCIikpMUUCIikpMUUCIikpMUUCIikpMUUCIikpMUUCIikpMU\nUCIikpMyCigzu8jMXvv/27vzKDnK897j36d7ehbNopFG+74gEBJagEHIFthwARvFNrJj9sXATULs\nhHOznGOs+Dpgk+QcjJ1w7/WF2NyExQ62hLENimNMAMnYOmHRgMUiCQktCEbraIRGM9Js3f3cP6p6\npmc0mzQtTXfr9zmnTlW99dbb79s1U0+/1dVvmdlWM1vew/YiM1sZbn/VzKYN5vVEROT0ccIBysyi\nwIPAUmAOcIOZzemW7Y+Aj9z9DOAB4Nsn+noiInJ6GUwPahGw1d23u3sbsAJY1i3PMuDxcPkp4DIz\ns0G8poiInCYG80TdiQ7NEKcAACAASURBVMCHaeu1wIW95XH3uJk1AFXAgfRMZnYHcEe42mRmmwdR\nr5RR3V8nD+Rjm0DtyiX52CbIz3Zlc5umDiTToB75ninu/jDwcCbLNLMad6/OZJlDLR/bBGpXLsnH\nNkF+tisf2jSYS3y7gMlp65PCtB7zmFkBMByoH8RriojIaWIwAWodMMvMpptZIXA9sKpbnlXAreHy\n1cBqd/dBvKaIiJwmTvgSX/id0p3Ac0AUeMTdN5jZvUCNu68C/hX4kZltBQ4SBLFTJaOXDLNEPrYJ\n1K5cko9tgvxsV863ydShERGRbKSRJEREJCspQImISFZSgBIRkaykACUiIllJAUpERLKSApSIiGQl\nBSgREclKClAiIpKVFKBERCQrKUCJiEhWUoASEZGspAAlIiJZqd8AZWaPmNl+M3unl+1mZv/HzLaa\n2Vtmdl7atlvN7L1wurWn/UVERHoykB7UY8CVfWxfCswKpzuAfwYws5HAPQSPgV8E3GNmIwZTWRER\nOX30+zwod/+tmU3rI8sy4IfhgwhfMbNKMxsPXAI87+4HAczseYJA95O+Xm/UqFE+bVpfLyciIrns\n9ddfP+Duo/vLd8IPLEwzEfgwbb02TOst/RhmdgdB74spU6ZQU1OTgWqJiEg2MrOdA8mXFTdJuPvD\n7l7t7tWjR/cbVEVE5DSQiQC1C5ictj4pTOstXUREpF+ZCFCrgC+Fd/MtBhrcfQ/wHPApMxsR3hzx\nqTBNRESkX/1+B2VmPyG44WGUmdUS3JkXA3D37wO/Av4A2AocBW4Ptx00s78D1oVF3Zu6YUJEJJu1\nt7dTW1tLS0vLUFclpxUXFzNp0iRisdgJ7W/BzXfZo7q62nWThIgMpR07dlBeXk5VVRVmNtTVyUnu\nTn19PY2NjUyfPr3LNjN73d2r+ysjK26SEBHJJi0tLQpOg2RmVFVVDaoXqgAlItIDBafBG+x7qAAl\nIiJZSQFKRESykgKUiEiWOXToEA899FBGy7zyyitZsGABc+fO5ctf/jKJRCKj5Z8MClAiIlnmZASo\nJ598kjfffJN33nmHuro6fvrTn2a0/JMhE2PxiYjkrW/9+wY27j6c0TLnTKjgns/N7XX78uXL2bZt\nGwsXLuSKK64A4Nlnn8XM+MY3vsF1113Hb37zG+6++27Ky8vZunUrl156KQ899BCRSM/9joqKCgDi\n8ThtbW05cROIelAiIlnmvvvuY+bMmaxfv57Fixezfv163nzzTV544QW++tWvsmfPHgBee+01vve9\n77Fx40a2bdvGz3/+8z7L/fSnP82YMWMoLy/n6quvPhVNGRT1oERE+tBXT+dUWLt2LTfccAPRaJSx\nY8fyyU9+knXr1lFRUcGiRYuYMWMGADfccANr167tM/A899xztLS0cNNNN7F69eqO3lm2Ug9KRCRH\ndb9MN5DLdsXFxSxbtoxnnnnmZFUrYxSgRESyTHl5OY2NjQBcfPHFrFy5kkQiQV1dHb/97W9ZtGgR\nEFzi27FjB8lkkpUrV3LRRRf1WF5TU1PHZcF4PM5//Md/MHv27FPTmEHQJT4RkSxTVVXFkiVLOOec\nc1i6dCnz589nwYIFmBn3338/48aN49133+WCCy7gzjvv7LhJ4gtf+EKP5R05coSrrrqK1tZWkskk\nl156KV/+8pdPcauOnwKUiEgW+vGPf9xl/Tvf+c4xeSoqKvjlL3/Zb1ljx45l3bp1/ebLNrrEJyIi\nWUk9KBGRHHTJJZdwySWXHJN+4YUX0tra2iXtRz/6EfPmzTtFNcscBSgRkTzy6quvDnUVMkaX+ERE\nJCspQImISFZSgBIRkaw0oABlZlea2WYz22pmy3vY/oCZrQ+nLWZ2KG1bIm3bqkxWXkRE8le/AcrM\nosCDwFJgDnCDmc1Jz+Puf+XuC919IfA9IH3EwubUNne/KoN1FxERoKysrNdt77///jG/qRqslpYW\nFi1a1PF8qXvuuSej5acM5C6+RcBWd98OYGYrgGXAxl7y3wCcnNqKiJxqzy6HvW9ntsxx82DpfZkt\nsxepAHXjjTdmrMyioiJWr15NWVkZ7e3tXHTRRSxdupTFixdn7DVgYJf4JgIfpq3XhmnHMLOpwHRg\ndVpysZnVmNkrZvb5Xva7I8xTU1dXN8Cqi4jkp+XLl/Pggw92rH/zm9/k7//+77nssss477zzmDdv\n3oAHe12+fDm/+93vWLhwIQ888AAtLS3cfvvtzJs3j3PPPZc1a9YA8Nhjj7Fs2TIuueQSZs2axbe+\n9a1eyzSzjl5be3s77e3tJ+f5Uu7e5wRcDfxL2votwP/tJe/XgO91S5sYzmcA7wMz+3q9888/30VE\nhtLGjRuH9PXfeOMN/8QnPtGxfvbZZ/sHH3zgDQ0N7u5eV1fnM2fO9GQy6e7upaWlvZa1Zs0a/8xn\nPtOx/t3vftdvv/12d3fftGmTT5482Zubm/3RRx/1cePG+YEDB/zo0aM+d+5cX7duXa/lxuNxX7Bg\ngZeWlvpdd93Va76e3kugxvuJPe4+oB7ULmBy2vqkMK0n1wM/6RYAd4Xz7cBvgHMH8JoiIqetc889\nl/3797N7927efPNNRowYwbhx4/j617/O/Pnzufzyy9m1axf79u077rLXrl3LzTffDMDs2bOZOnUq\nW7ZsAeCKK66gqqqKkpIS/vAP/5C1a9f2Wk40GmX9+vXU1tby2muv8c4775xYY/swkO+g1gGzzGw6\nQWC6HjjmYqaZzQZGAC+npY0Ajrp7q5mNApYA92ei4iIi+eyaa67hqaeeYu/evVx33XU88cQT1NXV\n8frrrxOLxZg2bRotLS0Zfc0Teb5UZWUll156Kb/+9a8555xzMlqffntQ7h4H7gSeAzYBT7r7BjO7\n18zS78q7HlgRdt9SzgZqzOxNYA1wn7v3dnOFiIiErrvuOlasWMFTTz3FNddcQ0NDA2PGjCEWi7Fm\nzRp27tw5oHLSny0FwfOlnnjiCQC2bNnCBx98wFlnnQXA888/z8GDB2lububpp59myZIlPZZZV1fH\noUPBr4mam5t5/vnnT8rzpQY0Fp+7/wr4Vbe0u7utf7OH/f4LyL0RCkVEhtjcuXNpbGxk4sSJjB8/\nnptuuonPfe5zzJs3j+rq6gEHhPnz5xONRlmwYAG33XYbf/Znf8ZXvvIV5s2bR0FBAY899hhFRUUA\nLFq0iC9+8YvU1tZy8803U11d3WOZe/bs4dZbbyWRSJBMJrn22mv57Gc/m7G2p2iwWBGRLPX22523\nt48aNYqXX365x3xNTU29lhGLxVi9enWXtEcffbTHvJMmTeLpp5/ut17z58/n97//fb/5BktDHYmI\nSFZSD0pEJA+8/fbb3HLLLV3SioqKBvz4jdtuu43bbrutS1p9fT2XXXbZMXlffPFFqqqqTriuA6UA\nJSLSA3c/OT8+PUnmzZvH+vXrM1pmVVXVoMrses/c8dMlPhGRboqLi6mvrx/0CfZ05u7U19dTXFx8\nwmWoByUi0s2kSZOora1FQ68NTnFxMZMmTTrh/RWgRES6icViTJ8+faircdrTJT4REclKClAiIpKV\nFKBERCQrKUCJiEhWUoASEZGspAAlIiJZSQFKRESykgKUiIhkJQUoERHJSgpQIiKSlRSgREQkKylA\niYhIVhpQgDKzK81ss5ltNbPlPWy/zczqzGx9OP1x2rZbzey9cLo1k5UXEZH81e9o5mYWBR4ErgBq\ngXVmtsrdN3bLutLd7+y270jgHqAacOD1cN+PMlJ7ERHJWwPpQS0Ctrr7dndvA1YAywZY/qeB5939\nYBiUngeuPLGqiojI6WQgAWoi8GHaem2Y1t0XzewtM3vKzCYfz75mdoeZ1ZhZjR4QJiIikLmbJP4d\nmObu8wl6SY8fz87u/rC7V7t79ejRozNUJRERyWUDCVC7gMlp65PCtA7uXu/ureHqvwDnD3RfERGR\nngwkQK0DZpnZdDMrBK4HVqVnMLPxaatXAZvC5eeAT5nZCDMbAXwqTBMREelTv3fxuXvczO4kCCxR\n4BF332Bm9wI17r4K+B9mdhUQBw4Ct4X7HjSzvyMIcgD3uvvBk9AOERHJM+buQ12HLqqrq72mpmao\nqyEiIieJmb3u7tX95dNIEiIikpUUoEREJCspQImISFbq9yYJETm9tcWTNLclaG5P0J5IEotGiEWN\nWEGEwmiEWDRCNGK97u/utCWStLQlaW4PyjnaFqelPUFzW5KW9gTRiFFYEKGoIEJhaop2LhcVRCkq\niFAQMZIOSXeS7iSSTjIZrCfcSSbDuUMy6cSTQZ5E0oknkySTEE8mO9ISYZ6kOx7WNZkMxmVLuuMe\npKWvRyNGQcQ62l0QNQoikXAeLMeiRjRiRCyYm9FlOWrBtkjEiBgkk9AaT9DSnuxz3hpPYkBBNHgv\nUq8dixoF0QixiIV1CtIiHa9D57JxTHqqTan9YmH5Zr0f11NBAUpkiLQnkjS2xGluTxALTw4FUSOW\ndrLr7QTRnkhytDXBkbY4R9viHAmXm9sSHGlLcLQ1Tms8SXsiSWs8SVu43DFPJGmLezhPcLQtQXNb\nOA8DSCotnuz/RqqIQSwaBqyC4CSXSHpHYBtAEZKFUkEwFgmOa0HEuPSsMXz76vmn5vVPyauInEKp\nkzdGl0+L1rFMx7qZEU8kOdKaoLG1nabWOE0tcRrDeff1pHtHGdFI8Ak4mioztRz2Jo60xjnc0k5j\nS5zDze0cDueNLUH60bZEv22JpX06L4xGiIcn/bZE8rjfl9Qn48KCSEcwKQqXSwqjDCuMUjksRklh\nAcNi0Y60YYVRSgoLKIlFiUWNeNLTgl2wnAp67fHObdGoURKLBlNh13lx2nJRQYSEO21hIG2LpwJo\n0GsI5kFaPOFd32tLHQPC3kiq5xIc41g0QiTsHUTDfaLRbusR69jXSP/bACPoZRhBmQAJd+KJoOcV\nTyTDudOeTJJIBD21VFoy1ZsLe3dJD/b3VO8v7KGZGcWxoKfY17ywIPhWJh6+76nXiSeDY9FRn7Bu\n3V871fNMhj3FVF062tBRZpK2tPJSxziecM4eX3Hcf3snSgFKsp67s+tQMxt2H6a+qY2G5nYONbfR\ncLQ9WA7nqampNT7gss1gIL+0MIPSwgIiYf5EeIJJX+6uIGJUlMSoKC6gvDhGRUkBY8rLqCiOUV5c\n0LGtpDAanATiqZNBeKJIJGkPTxaptIgZpUUFlBZGGdZ9XlhAaVFnQCku6BqMUoFTJFcoQEnW2dvQ\nwlu1h3h7VwNv1Tbw9q4GDh5p65KnMBph+LAYlSUxKofFmFBZzNnjKxgerpcWBX/a7l0/xXr43UTn\nuhOJGOXFMcqLCigrLqAsnJcXBYGlrDjoUfR3gk+G32UkwtcpKogM+TV8kVymACVDJp5Isr+xlc17\nG8NAdIi3ahvY3xgM6xiNGLPGlHH52WOYN6mScyZUMH54CcNLYhTHsu/kH4kYEUz/VCIZov8lOSkS\nSWd/Ywt7GlrYc6iFPQ3N7GloYW9DC7sbmtlzqIX9jS0dX56bwczRZVx0xijmTRrO/EmVzBlfQUlh\ndGgbIiJDRgFKBiWRdN6vP8KWvY28u7eRzXsb2byvkQ8OHj3me5niWIQJw0sYX1nMkjNGMaGymHHD\ni5k5uoxzJg6nrEh/jiLSSWcEGRB3Z39jaxiEDvPu3ka27GvkvX1NtMaDO8rMYFpVKWeOLWPpOeOY\nUFkSBKGKYD68JJZ1l+VEJHspQMkxkklnR/0RNuw+zIbdDWzcfZiNuw9Tn3ajwpjyIs4aV84ti6dy\n1rhyZo+r4IwxZbokJyIZowB1mmuNJ3hvXxMbdjeEAekwm/Yc7viNTixqnDm2nMvOHsOc8RXMHl/B\nWWPLGVFaOMQ1F5F8pwB1GqprbOXFTft4YdM+fvfegY5LdKWFUeZMqODa6snMmVDB3AkVzBpT3vHj\nQBGRU0kB6jTg7ry3v4nnNwZBaf2Hh3CHiZUlXH/BZC6YPpK5E4YzdeQw/ZhTRLKGAlSeiieSrHv/\nI14Ie0o7648CsGDScP768jO5fM5YZo8r100LIpK1FKDyzJHWOP/4n1v42Ru1NDS3U1gQYcnMKu74\nxAwumz2WccOLh7qKJ487JNqCKd4Gidbel+NhvkQrJBNdy+ha6LGvY5Fg6li2cN06t5mBJ4Mpmehc\nTp860hOdy8kEJOPd0uLhcljPSEEwWRQi0c71SNq6RYP8ifawzPZwOR7OU8vx8PWSne1172x3arm3\n8aCO+YCTvh7u5x6W72ntT1vu8h5bt3Kt63L3OvY4T6t3b++9e/D+dNQh/bha2nG1Y48r3drc5T04\n0W3pb9sA/gZJr1v6e9Y9faAfQLsfG+/5fcNh8oVw6dcHWO7gKEDlkZe31XPXz96k9qNmrlowgaXn\njOPiWaM7hv3JaskEHK2HI3XQtD+Yd18+cgDam8PA0t4t6LQFJ918ZZEg6EDniXUwIjGIxjoDW+rE\nDMee+Ho92XU7cXY5sTrHnNh7C+Sp8jv27xZkupTt9HgS7m0eCd+3jtdLm1Jt7xL40k7QyUS3tGQP\n73tam/sKLP0FHafb29tHMOsIxqk69xOsByr97yD9g1eXYxgJ/tdOkQGduczsSuB/A1HgX9z9vm7b\n/xr4YyAO1AH/3d13htsSwNth1g/c/aoM1V1CR1rjfPvX7/LDl3cyrWoYT/7px7hg2sjMFN7aBHWb\nYf9G+Oj94KRWUAyxYRArhoKSnucALQ3QfAhaDh07T9925EAQnHr6pBiJQeloKBsNw0bB8IkQLew6\nFRQF9YqG84KiHrZ3y5e+HCno/dMs9HBy4NieQE8nskj6iTFtORI5Nj0STesRpS8XpJ3I0yTTel7J\neNdeVmrdomlBqCBcjoXl69KuZL9+A5SZRYEHgSuAWmCdma1y941p2X4PVLv7UTP7CnA/cF24rdnd\nF2a43hJ6ZXs9X30q6DXdvmQad3169on9FineBvXvwf5NsG9DMN+/EQ7t7MxjkcF/ci8aDiXDobgS\niofDqFkwZTGUjoGyMVA6KlhOBaXiSp1MexKJAJEg6IjkqYH0oBYBW919O4CZrQCWAR0Byt3XpOV/\nBbg5k5WUYx1ti/PtZ9/l8Zd3MrVqGCvv+BiLpoe9pgNb4cNXId4C8dbwO5fWzvX0eXszHNwO9VuD\nT90QfNqumgUTz4dzb4ExZwfTiGnB9ngLtLdAvDnYv725s6zUHO8MQiWVncsR/ZBXRAZmIAFqIvBh\n2notcGEf+f8IeDZtvdjMaggu/93n7k8fdy2li1e213PXU2/xwcGj3Pbxadx15VkMKwwP5c7/gn+7\nGtqPHLtjtLDz8lZBced85AyY/RkYMyeYqs6Agj5+iFtYGkwiIidRRr89N7ObgWrgk2nJU919l5nN\nAFab2dvuvq3bfncAdwBMmTIlk1XKK0fb4tz/68089l/vM2XkMFbesZgLZ1R1Znh/LTxxbfA9zTWP\nw7CqINAUFIfftegHtyKSOwYSoHYBk9PWJ4VpXZjZ5cD/BD7p7q2pdHffFc63m9lvgHOBLgHK3R8G\nHgaorq4+zltPcpu709yeoKklzuG0R4w3tQaPCO947HhrnOc27GVnfQ+9JoAdv4UfXweVU+BLq6B8\n7NA1SkQkAwYSoNYBs8xsOkFguh64MT2DmZ0L/AC40t33p6WPAI66e6uZjQKWENxAcVpxdw4eaWP7\ngSPsqDvCtgNNbK87wva6Jj44eJT2RP8xuTgWYerIUn7yJ4v52Myqrhu3vxQEpxHT4NZVwc0GIiI5\nrt8A5e5xM7sTeI7gNvNH3H2Dmd0L1Lj7KuA7QBnw03BkgtTt5GcDPzCzJBAh+A5qY48vlAcSSWdn\n/RG27GtiW10YhMJg1NDc+RudwmiEqVXDOGNMGZfPGcuIYYWUFRVQXhxMZUWxLuulRQXEor1cntu2\nBn5yffA90pdWBXe+iYjkAfPj/THXSVZdXe01NTVDXY0+JZLOBwePhs9DauS9/U0dQakt3nkb9pjy\nImaMLmXG6DJmjCpl5ugyZowuZdKIYUQzMebd1hdhxY3BTQ1feia4RVtEJMuZ2evuXt1fvhwYYmBo\nJZLOpj2HeXXHQd6uPdQRiFrTAtHEyhJmjS3j4lmjmDWmjDPHljNzTNnJfULsey8EwWnUmWFwqup/\nHxGRHKIA1U0i6WzcfZhXd9TzyvZ6XttxkMMtwe+Dxg8v5syx5Sw5o4pZY8s5c2w5Z/QUiNqbIdkM\nydKTc+fclv+ElTfB6LOCy3rDMjRqhIhIFjntA1Q8kWRDR0A6yLr3D9IYBqTpo0r5g3njWTyjigtn\njGT88JJjC2g7Ah/UwO71sPv3sGd9MDRQatieWCkUlYW/HSoLp7S0ouFQNSP4/dHo2f0Hm82/hifD\nH8/e8rSCk4jkrdM2QMUTSb7/0jZ+8NJ2GluDgDRjdCmfnT+BxTNGsnhGFWMruo383XYU9r0TBKJU\nQDqwuXP4n7KxMOFcmPN5KBwWBK+2I9Da2Lnc1gRHDwTj2rUdCcaiaz/a+Rrl44PgMzocvWHMnKCn\nVFQGm5+FlbfA2LnwpaehZMSpebNERIbAaRmgtu47zD+sWE3D3h3cOTnJ4okFzKhwymmG1sOw8zBs\naQwCS2sjtBwO5k37Oh95UDomDEZXBfPxC6Fi/PFXxh0aaqHu3WDsu9QYeDWPBEMJpVROgcN7YNw8\nuOUXwfBBIiJ5LD8DVCIOjbvh0Idw6ANo+BAO7cQPfUjj3u1MObqHRy0ORcD+cEopLIOiCigq75zK\nx0NxBZRPgAkLg4BUPj4zg5iaQeXkYJp1RWd6MhH0svZvgrpNwTxaCFfep+AkIqeF/AtQB96DBy/s\n7OmEEsPGsK19JJtbJhId8XEuvuA8ysfNhIoJwSCmReVBcMqWwUwjUaiaGUxnf3aoayMicsrlX4Aq\nHw8X/VVwSaxyMj58Ck++59z77DYiZty9bA5Xnz9JjzoXEcly+Regisrgsr8FYG9DC1/72Vu8tKWO\nj8+s4jvXLGBiZQ934omISNbJvwBFMPbdM+t3c/cz79CWSHLvsrncfOFUIpkYvUFERE6JvAtQ9U2t\nfOPpd3j2nb2cN6WSf7x2IdNH6dlFIiK5Ju8CVHN7gtd2HGT50tn8ycUzMjPmnYiInHJ5F6AmjRjG\n7752addnJYmISM7Jy0esKjiJiOS+vAxQIiKS+xSgREQkKylAiYhIVlKAEhGRrKQAJSIiWWlAAcrM\nrjSzzWa21cyW97C9yMxWhttfNbNpadv+JkzfbGafzlzVRUQkn/UboMwsCjwILAXmADeY2Zxu2f4I\n+MjdzwAeAL4d7jsHuB6YC1wJPBSWJyIi0qeB9KAWAVvdfbu7twErgGXd8iwDHg+XnwIus2C48GXA\nCndvdfcdwNawPBERkT4NJEBNBD5MW68N03rM4+5xoAGoGuC+IiIix8iKmyTM7A4zqzGzmrq6uqGu\njoiIZIGBjAm0C5ictj4pTOspT62ZFQDDgfoB7ou7Pww8DGBmdWa2c6AN6MMo4EAGyskm+dgmULty\nST62CfKzXdncpqkDyTSQALUOmGVm0wmCy/XAjd3yrAJuBV4GrgZWu7ub2Srgx2b2T8AEYBbwWl8v\n5u6jB1Lx/phZjbtXZ6KsbJGPbQK1K5fkY5sgP9uVD23qN0C5e9zM7gSeA6LAI+6+wczuBWrcfRXw\nr8CPzGwrcJAgiBHmexLYCMSBP3f3xElqi4iI5JEBDfvt7r8CftUt7e605Rbgml72/QfgHwZRRxER\nOQ1lxU0SJ8nDQ12BkyAf2wRqVy7JxzZBfrYr59tk7j7UdRARETlGPvegREQkhylAiYhIVsq7ANXf\nwLa5yszeN7O3zWy9mdUMdX1OlJk9Ymb7zeydtLSRZva8mb0XzkcMZR2PVy9t+qaZ7QqP13oz+4Oh\nrOOJMLPJZrbGzDaa2QYz+4swPWePVx9tyunjZWbFZvaamb0ZtutbYfr0cADvreGA3oVDXdfjkVff\nQYUD0W4BriAYVmkdcIO7bxzSimWAmb0PVLt7tv7wbkDM7BNAE/BDdz8nTLsfOOju94UfKka4+9eG\nsp7Ho5c2fRNocvfvDmXdBsPMxgPj3f0NMysHXgc+D9xGjh6vPtp0LTl8vMKxT0vdvcnMYsBa4C+A\nvwZ+7u4rzOz7wJvu/s9DWdfjkW89qIEMbCtDyN1/S/BbuXTpgw0/TnDCyBm9tCnnufsed38jXG4E\nNhGMpZmzx6uPNuU0DzSFq7FwcuC/EQzgDTl2rCD/AlQ+D07rwH+a2etmdsdQVybDxrr7nnB5LzB2\nKCuTQXea2VvhJcCcuQzWk/AZb+cCr5Inx6tbmyDHj5eZRc1sPbAfeB7YBhwKB/CGHDwf5luAymcX\nuft5BM/l+vPwslLe8eCacz5cd/5nYCawENgD/OPQVufEmVkZ8DPgL939cPq2XD1ePbQp54+Xuyfc\nfSHBmKeLgNlDXKVBy7cANaDBaXORu+8K5/uBX5Bfz9XaF343kPqOYP8Q12fQ3H1feMJIAv+PHD1e\n4fcZPwOecPefh8k5fbx6alO+HC8Adz8ErAE+BlSGA3hDDp4P8y1AdQxsG96tcj3BQLY5zcxKwy90\nMbNS4FPAO33vlVNSgw0Tzp8ZwrpkROoEHvoCOXi8wi/e/xXY5O7/lLYpZ49Xb23K9eNlZqPNrDJc\nLiG4UWwTQaC6OsyWU8cK8uwuPoDw9tD/RefAtjk/DqCZzSDoNUEwfuKPc7VdZvYT4BKCRwHsA+4B\nngaeBKYAO4Fr3T1nbjropU2XEFwucuB94E/TvrfJCWZ2EfA74G0gGSZ/neA7m5w8Xn206QZy+HiZ\n2XyCmyCiBB2PJ9393vDcsQIYCfweuNndW4eupscn7wKUiIjkh3y7xCciInlCAUpERLKSApSIiGQl\nBSgREclKClAiIpKVFKBEcoCZXWJmvxzqeoicSgpQIiKSlRSgRDLIzG4On8uz3sx+EA7g2WRmD4TP\n6XnRzEaHeReaAuYGNAAAAX5JREFU2SvhAKW/SA1QamZnmNkL4bN93jCzmWHxZWb2lJm9a2ZPhKMi\niOQtBSiRDDGzs4HrgCXhoJ0J4CagFKhx97nASwQjTQD8EPiau88nGNkglf4E8KC7LwA+TjB4KQQj\nb/8lMAeYASw56Y0SGUIF/WcRkQG6DDgfWBd2bkoIBlJNAivDPP8G/NzMhgOV7v5SmP448NNwzMWJ\n7v4LAHdvAQjLe83da8P19cA0ggfTieQlBSiRzDHgcXf/my6JZn/bLd+Jji+WPoZaAv3/Sp7TJT6R\nzHkRuNrMxgCY2Ugzm0rwf5YaUfpGYK27NwAfmdnFYfotwEvhU15rzezzYRlFZjbslLZCJEvoE5hI\nhrj7RjP7BsGTjyNAO/DnwBFgUbhtP8H3VBA8/uD7YQDaDtwept8C/MDM7g3LuOYUNkMka2g0c5GT\nzMya3L1sqOshkmt0iU9ERLKSelAiIpKV1IMSEZGspAAlIiJZSQFKRESykgKUiIhkJQUoERHJSv8f\nFxfdf7aRXZgAAAAASUVORK5CYII=\n",
"text/plain": [
"<matplotlib.figure.Figure at 0x7f5a150be1d0>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"plot_history(old_hist)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-21T22:18:36.696824Z",
"start_time": "2017-11-21T22:18:36.613818Z"
}
},
"outputs": [],
"source": [
"epochs = 50\n",
"batch_size = 100\n",
"\n",
"model2.compile(\n",
" loss='sparse_categorical_crossentropy',\n",
" optimizer=keras.optimizers.Nadam(lr=0.001),\n",
" metrics=['accuracy', top_3])\n",
"\n",
"model2.fit(\n",
" generator=img_gen.flow(x_train, y_train, batch_size=batch_size),\n",
" steps_per_epoch=len(x_train) / batch_size,\n",
" epochs=epochs,\n",
" validation_data=img_gen.flow(x_test, y_test, batch_size=batch_size),\n",
" validation_steps=len(x_test) / batch_size,\n",
" callbacks=[keras.callbacks.EarlyStopping(patience=4)]\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 92,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-21T22:24:11.407289Z",
"start_time": "2017-11-21T22:23:59.381258Z"
}
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 95,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-21T22:29:33.525289Z",
"start_time": "2017-11-21T22:29:21.812957Z"
}
},
"outputs": [],
"source": [
"import keras.preprocessing.image\n",
"img_gen = keras.preprocessing.image.ImageDataGenerator(\n",
" featurewise_center=True,\n",
" featurewise_std_normalization=True,\n",
" zca_whitening=True,\n",
" zca_epsilon=1e-6,\n",
" width_shift_range=4/32,\n",
" height_shift_range=4/32,\n",
" fill_mode='constant',\n",
" cval=0.,\n",
" horizontal_flip=True,\n",
" data_format=K.image_data_format())\n",
"img_gen.fit(x_train)"
]
},
{
"cell_type": "code",
"execution_count": 100,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-21T22:32:45.380141Z",
"start_time": "2017-11-21T22:32:45.161843Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"input_37 (InputLayer) (None, 32, 32, 3) 0 \n",
"_________________________________________________________________\n",
"conv2d_84 (Conv2D) (None, 32, 32, 128) 1664 \n",
"_________________________________________________________________\n",
"max_pooling2d_82 (MaxPooling (None, 16, 16, 128) 0 \n",
"_________________________________________________________________\n",
"batch_normalization_84 (Batc (None, 16, 16, 128) 512 \n",
"_________________________________________________________________\n",
"dropout_113 (Dropout) (None, 16, 16, 128) 0 \n",
"_________________________________________________________________\n",
"conv2d_85 (Conv2D) (None, 16, 16, 128) 65664 \n",
"_________________________________________________________________\n",
"max_pooling2d_83 (MaxPooling (None, 8, 8, 128) 0 \n",
"_________________________________________________________________\n",
"batch_normalization_85 (Batc (None, 8, 8, 128) 512 \n",
"_________________________________________________________________\n",
"dropout_114 (Dropout) (None, 8, 8, 128) 0 \n",
"_________________________________________________________________\n",
"conv2d_86 (Conv2D) (None, 8, 8, 128) 65664 \n",
"_________________________________________________________________\n",
"max_pooling2d_84 (MaxPooling (None, 4, 4, 128) 0 \n",
"_________________________________________________________________\n",
"batch_normalization_86 (Batc (None, 4, 4, 128) 512 \n",
"_________________________________________________________________\n",
"dropout_115 (Dropout) (None, 4, 4, 128) 0 \n",
"_________________________________________________________________\n",
"flatten_49 (Flatten) (None, 2048) 0 \n",
"_________________________________________________________________\n",
"dense_77 (Dense) (None, 200) 409800 \n",
"_________________________________________________________________\n",
"dropout_116 (Dropout) (None, 200) 0 \n",
"_________________________________________________________________\n",
"dense_78 (Dense) (None, 100) 20100 \n",
"=================================================================\n",
"Total params: 564,428\n",
"Trainable params: 563,660\n",
"Non-trainable params: 768\n",
"_________________________________________________________________\n"
]
}
],
"source": [
"dropout_prob = .4\n",
"model2 = keras.Sequential([\n",
" keras.layers.InputLayer(input_shape=x_test[0, ...].shape),\n",
"\n",
" #Conv1\n",
" keras.layers.Conv2D(128, (2, 2), activation='elu', padding=\"same\"),\n",
" keras.layers.MaxPooling2D(\n",
" pool_size=(2, 2), strides=(2, 2), padding=\"same\"),\n",
" keras.layers.BatchNormalization(),\n",
" keras.layers.Dropout(dropout_prob),\n",
"\n",
" #Conv2\n",
" keras.layers.Conv2D(128, (2, 2), activation='elu', padding=\"same\"),\n",
" keras.layers.MaxPooling2D(\n",
" pool_size=(2, 2), strides=(2, 2), padding=\"same\"),\n",
" keras.layers.BatchNormalization(),\n",
" keras.layers.Dropout(.7 * dropout_prob),\n",
"\n",
" #Conv3\n",
" keras.layers.Conv2D(128, (2, 2), activation='elu', padding=\"same\"),\n",
" keras.layers.MaxPooling2D(\n",
" pool_size=(2, 2), strides=(2, 2), padding=\"same\"),\n",
" keras.layers.BatchNormalization(),\n",
" keras.layers.Dropout(.5 * dropout_prob),\n",
"\n",
" #Dense1\n",
" keras.layers.Flatten(),\n",
" keras.layers.Dense(200, activation=\"elu\"),\n",
" keras.layers.Dropout(dropout_prob),\n",
"\n",
" #Class\n",
" keras.layers.Dense(num_class, activation=\"softmax\"),\n",
"])\n",
"model2.summary()"
]
},
{
"cell_type": "code",
"execution_count": 208,
"metadata": {
"ExecuteTime": {
"end_time": "2017-11-22T22:49:21.559748Z",
"start_time": "2017-11-22T22:49:21.488562Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"training according to schedule [ 1.00000000e-01 3.59381366e-02 1.29154967e-02 4.64158883e-03\n",
" 1.66810054e-03 5.99484250e-04 2.15443469e-04 7.74263683e-05\n",
" 2.78255940e-05 1.00000000e-05]\n",
"training at lr=0.1\n",
"Train on 100000 samples, validate on 10000 samples\n"
]
},
{
"ename": "TypeError",
"evalue": "'float' object cannot be interpreted as an integer",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-208-82ac53b34377>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprev\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msub_history\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m \u001b[0mfit_sched\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m<ipython-input-208-82ac53b34377>\u001b[0m in \u001b[0;36mfit_sched\u001b[0;34m(model, lr_sched)\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m callbacks=[keras.callbacks.EarlyStopping(patience=3, verbose=1)])\n\u001b[0m\u001b[1;32m 23\u001b[0m \u001b[0mnum_epochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msub_history\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"loss\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0mcurr_epoch\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mnum_epochs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/models.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m 958\u001b[0m \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 959\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 960\u001b[0;31m validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m 961\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 962\u001b[0m def evaluate(self, x, y, batch_size=32, verbose=1,\n",
"\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m 1648\u001b[0m \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1649\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1650\u001b[0;31m validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m 1651\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1652\u001b[0m def evaluate(self, x=None, y=None,\n",
"\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m 1158\u001b[0m \u001b[0mcbk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1160\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1161\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_epoch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1162\u001b[0m \u001b[0mepoch_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mTypeError\u001b[0m: 'float' object cannot be interpreted as an integer"
]
}
],
"source": [
"def fit_sched(model, lr_sched=np.logspace(-1, -5, 10)):\n",
" \n",
" print(\"training according to schedule {}\".format(lr_sched))\n",
"\n",
" curr_epoch = 0\n",
" history = {}\n",
" \n",
" for lr in lr_sched:\n",
" \n",
" print(\"training at lr={}\".format(lr))\n",
" model.compile(\n",
" loss='sparse_categorical_crossentropy',\n",
" optimizer=keras.optimizers.Nadam(lr=lr),\n",
" metrics=['accuracy', top_3])\n",
" \n",
" sub_history = model1.fit(\n",
" x_train,\n",
" y_train,\n",
" initial_epoch=int(curr_epoch),\n",
" batch_size=320,\n",
" epochs=1e3,\n",
" verbose=1,\n",
" validation_data=(x_test, y_test),\n",
" callbacks=[keras.callbacks.EarlyStopping(patience=3, verbose=1)])\n",
" \n",
" num_epochs = len(sub_history.history[\"loss\"])\n",
" curr_epoch += num_epochs\n",
" \n",
" for key in sug_history.history.keys():\n",
" prev = history.get(key,[])\n",
" history[key] = prev + sub_history.history[key]\n",
" \n",
" return history\n",
"\n",
" fit_sched(model2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2017-11-21T22:32:46.220Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/50\n",
"500/500 [==============================] - 108s 217ms/step - loss: 4.2304 - acc: 0.0883 - top_3: 0.1940 - val_loss: 3.5486 - val_acc: 0.1769 - val_top_3: 0.3401\n",
"Epoch 2/50\n",
"500/500 [==============================] - 105s 211ms/step - loss: 3.3563 - acc: 0.1984 - top_3: 0.3757 - val_loss: 2.9361 - val_acc: 0.2784 - val_top_3: 0.4812\n",
"Epoch 3/50\n",
"500/500 [==============================] - 104s 208ms/step - loss: 3.0248 - acc: 0.2539 - top_3: 0.4534 - val_loss: 2.7481 - val_acc: 0.3146 - val_top_3: 0.5219\n",
"Epoch 4/50\n",
"500/500 [==============================] - 105s 210ms/step - loss: 2.8459 - acc: 0.2893 - top_3: 0.4953 - val_loss: 2.5345 - val_acc: 0.3551 - val_top_3: 0.5635\n",
"Epoch 5/50\n",
"500/500 [==============================] - 105s 210ms/step - loss: 2.7246 - acc: 0.3112 - top_3: 0.5218 - val_loss: 2.4734 - val_acc: 0.3672 - val_top_3: 0.5760\n",
"Epoch 6/50\n",
"500/500 [==============================] - 105s 211ms/step - loss: 2.6434 - acc: 0.3273 - top_3: 0.5401 - val_loss: 2.3766 - val_acc: 0.3819 - val_top_3: 0.6045\n",
"Epoch 7/50\n",
"500/500 [==============================] - 105s 210ms/step - loss: 2.5700 - acc: 0.3413 - top_3: 0.5573 - val_loss: 2.2993 - val_acc: 0.3958 - val_top_3: 0.6158\n",
"Epoch 8/50\n",
"500/500 [==============================] - 105s 209ms/step - loss: 2.5219 - acc: 0.3511 - top_3: 0.5669 - val_loss: 2.2572 - val_acc: 0.4150 - val_top_3: 0.6268\n",
"Epoch 9/50\n",
"500/500 [==============================] - 105s 209ms/step - loss: 2.4761 - acc: 0.3591 - top_3: 0.5791 - val_loss: 2.1776 - val_acc: 0.4241 - val_top_3: 0.6481\n",
"Epoch 10/50\n",
"500/500 [==============================] - 106s 212ms/step - loss: 2.4363 - acc: 0.3689 - top_3: 0.5860 - val_loss: 2.1624 - val_acc: 0.4297 - val_top_3: 0.6445\n",
"Epoch 11/50\n",
"500/500 [==============================] - 106s 211ms/step - loss: 2.4045 - acc: 0.3746 - top_3: 0.5976 - val_loss: 2.1401 - val_acc: 0.4358 - val_top_3: 0.6492\n",
"Epoch 12/50\n",
"500/500 [==============================] - 105s 209ms/step - loss: 2.3710 - acc: 0.3816 - top_3: 0.6016 - val_loss: 2.0806 - val_acc: 0.4543 - val_top_3: 0.6700\n",
"Epoch 13/50\n",
"500/500 [==============================] - 107s 213ms/step - loss: 2.3344 - acc: 0.3923 - top_3: 0.6118 - val_loss: 2.1015 - val_acc: 0.4445 - val_top_3: 0.6664\n",
"Epoch 14/50\n",
"500/500 [==============================] - 105s 210ms/step - loss: 2.3225 - acc: 0.3939 - top_3: 0.6132 - val_loss: 2.0271 - val_acc: 0.4643 - val_top_3: 0.6745\n",
"Epoch 15/50\n",
"500/500 [==============================] - 105s 211ms/step - loss: 2.2942 - acc: 0.3970 - top_3: 0.6201 - val_loss: 2.0450 - val_acc: 0.4603 - val_top_3: 0.6752\n",
"Epoch 16/50\n",
"500/500 [==============================] - 107s 213ms/step - loss: 2.2740 - acc: 0.4023 - top_3: 0.6242 - val_loss: 1.9890 - val_acc: 0.4697 - val_top_3: 0.6870\n",
"Epoch 17/50\n",
"500/500 [==============================] - 104s 209ms/step - loss: 2.2354 - acc: 0.4124 - top_3: 0.6340 - val_loss: 1.9804 - val_acc: 0.4688 - val_top_3: 0.6859\n",
"Epoch 19/50\n",
"500/500 [==============================] - 106s 213ms/step - loss: 2.2279 - acc: 0.4147 - top_3: 0.6352 - val_loss: 1.9484 - val_acc: 0.4770 - val_top_3: 0.6915\n",
"Epoch 20/50\n",
"390/500 [======================>.......] - ETA: 19s - loss: 2.2053 - acc: 0.4184 - top_3: 0.6419"
]
}
],
"source": [
"epochs = 50\n",
"batch_size = 100\n",
"\n",
"model2.compile(\n",
" loss='sparse_categorical_crossentropy',\n",
" optimizer=keras.optimizers.Nadam(lr=0.001),\n",
" metrics=['accuracy', top_3])\n",
"\n",
"model2.fit_generator(\n",
" generator=img_gen.flow(x_train, y_train, batch_size=batch_size),\n",
" steps_per_epoch=len(x_train) / batch_size,\n",
" epochs=epochs,\n",
" validation_data=img_gen.flow(x_test, y_test, batch_size=batch_size),\n",
" validation_steps=len(x_test) / batch_size,\n",
" callbacks=[keras.callbacks.EarlyStopping(patience=4)]\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
},
"toc": {
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"toc_cell": false,
"toc_position": {},
"toc_section_display": "block",
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment