Skip to content

Instantly share code, notes, and snippets.

@zackenton

zackenton/whitening.ipynb

Created January 4, 2018 10:58
Show Gist options
  • Save zackenton/30c76ce514f30240d680267b2e803a4e to your computer and use it in GitHub Desktop.
Save zackenton/30c76ce514f30240d680267b2e803a4e to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
whitening.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using TensorFlow backend.\n"
]
}
],
"source": [
"import tensorflow\n",
"import os\n",
"os.environ['KERAS_BACKEND'] = 'tensorflow'\n",
"import numpy as np\n",
"from keras.models import Sequential \n",
"from keras.layers import Dense, Activation, Reshape, Flatten\n",
"from keras.utils import np_utils \n",
"from keras import backend as K\n",
"from keras.datasets import mnist \n",
"from sklearn.model_selection import train_test_split\n",
"import matplotlib.pyplot as plt\n",
"from keras.callbacks import TensorBoard, LambdaCallback\n",
"from keras.utils import np_utils \n",
"from keras.preprocessing.image import ImageDataGenerator\n",
"import datetime\n",
"from keras.optimizers import SGD"
]
},
{
"cell_type": "code",
"execution_count": 120,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def compute_hessian_stats(epoch, logs):\n",
" if epoch%10==0:\n",
" print(epoch)\n",
" p = model.predict_proba(X_bin_train).reshape(-1,)\n",
" W = np.diag(p*(1-p))\n",
" H = X_bin_train.transpose().dot(W).dot(X_bin_train)\n",
" w, v = np.linalg.eig(H)\n",
" logs['largest_eigenvalue'] = np.max(w)\n",
" logs['det'] = np.prod(w)"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def compute_hessian_stats_2(epoch, logs):\n",
" if epoch%10==0:\n",
" print(epoch)\n",
" idx = np.random.randint(len(X_bin_train), size=3)\n",
" randX = X_bin_train[idx]\n",
"# randX = randX.reshape(len(randX),-1)\n",
"# print(randX)\n",
" p = model.predict_proba(randX).reshape(-1,)\n",
" W = np.diag(p*(1-p))\n",
"# H = randX.transpose().dot(W).dot(randX)\n",
"# w, v = np.linalg.eig(H)\n",
"# logs['largest_eigenvalue'] = np.max(w)\n",
"# logs['det'] = np.prod(w)"
]
},
{
"cell_type": "code",
"execution_count": 152,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(12665, 28, 28, 1)"
]
},
"execution_count": 152,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"X_bin_train.shape"
]
},
{
"cell_type": "code",
"execution_count": 162,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"flatten_11 (Flatten) (None, 784) 0 \n",
"_________________________________________________________________\n",
"dense_11 (Dense) (None, 1) 785 \n",
"=================================================================\n",
"Total params: 785\n",
"Trainable params: 785\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
}
],
"source": [
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": 165,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import keras.backend as K"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 163,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 0., 1., 1., 1., 1.], dtype=float32)"
]
},
"execution_count": 163,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"p = model.predict_proba(X_bin_train[:5])\n",
"p = p.reshape(-1,)\n",
"p"
]
},
{
"cell_type": "code",
"execution_count": 130,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 0., 1., 1., 1., 1.], dtype=float32)"
]
},
"execution_count": 130,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"p"
]
},
{
"cell_type": "code",
"execution_count": 119,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 2., 3.]], dtype=float32)"
]
},
"execution_count": 119,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"c = np.array([[2,3]]).astype(\"float32\")\n",
"c"
]
},
{
"cell_type": "code",
"execution_count": 124,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"c = c.flatten()"
]
},
{
"cell_type": "code",
"execution_count": 125,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 2., 0.],\n",
" [ 0., 3.]], dtype=float32)"
]
},
"execution_count": 125,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.diag(c)"
]
},
{
"cell_type": "code",
"execution_count": 164,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 0., 0., 0., 0., 0.],\n",
" [ 0., 1., 0., 0., 0.],\n",
" [ 0., 0., 1., 0., 0.],\n",
" [ 0., 0., 0., 1., 0.],\n",
" [ 0., 0., 0., 0., 1.]], dtype=float32)"
]
},
"execution_count": 164,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.diag(p)"
]
},
{
"cell_type": "code",
"execution_count": 134,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 0., 0., 0., 0., 0.],\n",
" [ 0., 0., 0., 0., 0.],\n",
" [ 0., 0., 0., 0., 0.],\n",
" [ 0., 0., 0., 0., 0.],\n",
" [ 0., 0., 0., 0., 0.]], dtype=float32)"
]
},
"execution_count": 134,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"W = np.diag(p*(1-p))\n",
"W"
]
},
{
"cell_type": "code",
"execution_count": 88,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([0, 1, 1, ..., 1, 0, 1], dtype=uint8)"
]
},
"execution_count": 88,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"y_bin_train"
]
},
{
"cell_type": "code",
"execution_count": 149,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[<tf.Tensor 'dense_4/Sigmoid:0' shape=(?, 1) dtype=float32>]"
]
},
"execution_count": 149,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.outputs"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Control"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# the data, shuffled and split between train and test sets \n",
"(X_train, y_train), (X_test, y_test) = mnist.load_data()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)\n",
"X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"input_dim = 784 #28*28 \n",
"X_train = X_train.astype('float32') \n",
"X_test = X_test.astype('float32') \n",
"X_train /= 255 \n",
"X_test /= 255"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# define data preparation\n",
"datagen = ImageDataGenerator()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"datagen.fit(X_train)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"nb_classes = 10\n",
"y_train = np_utils.to_categorical(y_train, nb_classes) \n",
"y_test = np_utils.to_categorical(y_test, nb_classes)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"input_dim = 784 #28*28 \n",
"output_dim = nb_classes = 10 \n",
"model = Sequential() \n",
"model.add(Flatten(input_shape=X_train.shape[1:]))\n",
"model.add(Dense(output_dim, activation='softmax')) \n",
"batch_size = 128 \n",
"nb_epoch = 20\n",
"lr = 0.1"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"name = datetime.datetime.now().isoformat() + \"control\"\n",
"c = {\"lr\": 0.1, \"bs\": batch_size, \"momentum\": 0.9}\n",
"name += \"lr=\" + str(c['lr'])\n",
"name += \"m=\" + str(c['momentum'])\n",
"name += \"bs=\" + str(c['bs'])"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"callbacks = []\n",
"callbacks.append(TensorBoard(log_dir=\"logreg_ex/\" + name))"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"model.compile(optimizer=SGD(lr=c['lr'], momentum=c['momentum']), loss='categorical_crossentropy', metrics=['accuracy']) "
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"<keras.callbacks.History at 0x127d8ecf8>"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),\n",
" steps_per_epoch=len(X_train) / batch_size, \n",
" epochs=nb_epoch, \n",
" verbose=0,\n",
" validation_data=(X_test, y_test), \n",
" callbacks=callbacks)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Whitened"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# the data, shuffled and split between train and test sets \n",
"(X_train, y_train), (X_test, y_test) = mnist.load_data()"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)\n",
"X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"input_dim = 784 #28*28 \n",
"X_train = X_train.astype('float32') \n",
"X_test = X_test.astype('float32') \n",
"X_train /= 255 \n",
"X_test /= 255"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# define data preparation\n",
"datagen = ImageDataGenerator(zca_whitening=True)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"datagen.fit(X_train)"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"nb_classes = 10\n",
"y_train = np_utils.to_categorical(y_train, nb_classes) \n",
"y_test = np_utils.to_categorical(y_test, nb_classes)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"input_dim = 784 #28*28 \n",
"output_dim = nb_classes = 10 \n",
"model = Sequential() \n",
"model.add(Flatten(input_shape=X_train.shape[1:]))\n",
"model.add(Dense(output_dim, activation='softmax')) \n",
"batch_size = 128 \n",
"nb_epoch = 20"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"name = datetime.datetime.now().isoformat() + \"whitened\"\n",
"c = {\"lr\": 0.1, \"bs\": 10, \"momentum\": 0.9}\n",
"name += \"lr=\" + str(c['lr'])\n",
"name += \"m=\" + str(c['momentum'])\n",
"name += \"bs=\" + str(c['bs'])"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"callbacks = []\n",
"callbacks.append(TensorBoard(log_dir=\"logreg_ex/\" + name))"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"model.compile(optimizer=SGD(lr=c['lr'], momentum=c['momentum']), loss='categorical_crossentropy', metrics=['accuracy']) "
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(10000, 28, 28, 1)\n",
"(10000, 28, 28, 1)\n"
]
}
],
"source": [
"# whiten test according to the training set whitening transform\n",
"print(X_test.shape)\n",
"for i in range(len(X_test)):\n",
" X_test[i] = datagen.standardize(X_test[i])\n",
"print(X_test.shape)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/20\n",
"469/468 [==============================] - 12s 27ms/step - loss: 0.7408 - acc: 0.8344 - val_loss: 0.6017 - val_acc: 0.8753\n",
"Epoch 2/20\n",
"469/468 [==============================] - 11s 23ms/step - loss: 0.5890 - acc: 0.8827 - val_loss: 0.5769 - val_acc: 0.8883\n",
"Epoch 3/20\n",
"469/468 [==============================] - 14s 30ms/step - loss: 0.5603 - acc: 0.8922 - val_loss: 0.5870 - val_acc: 0.8930\n",
"Epoch 4/20\n",
"469/468 [==============================] - 10s 22ms/step - loss: 0.5573 - acc: 0.8945 - val_loss: 0.5932 - val_acc: 0.8951\n",
"Epoch 5/20\n",
"469/468 [==============================] - 10s 21ms/step - loss: 0.5398 - acc: 0.8985 - val_loss: 0.5974 - val_acc: 0.8967\n",
"Epoch 6/20\n",
"469/468 [==============================] - 13s 29ms/step - loss: 0.5462 - acc: 0.8993 - val_loss: 0.5918 - val_acc: 0.8973\n",
"Epoch 7/20\n",
"469/468 [==============================] - 12s 25ms/step - loss: 0.5414 - acc: 0.9008 - val_loss: 0.6129 - val_acc: 0.8977\n",
"Epoch 8/20\n",
"469/468 [==============================] - 13s 29ms/step - loss: 0.5272 - acc: 0.9036 - val_loss: 0.5995 - val_acc: 0.8991\n",
"Epoch 9/20\n",
"469/468 [==============================] - 10s 20ms/step - loss: 0.5261 - acc: 0.9033 - val_loss: 0.5825 - val_acc: 0.9035\n",
"Epoch 10/20\n",
"469/468 [==============================] - 12s 25ms/step - loss: 0.5214 - acc: 0.9069 - val_loss: 0.5931 - val_acc: 0.9003\n",
"Epoch 11/20\n",
"469/468 [==============================] - 12s 25ms/step - loss: 0.5200 - acc: 0.9074 - val_loss: 0.6045 - val_acc: 0.9025\n",
"Epoch 12/20\n",
"469/468 [==============================] - 8s 17ms/step - loss: 0.5170 - acc: 0.9086 - val_loss: 0.6061 - val_acc: 0.9026\n",
"Epoch 13/20\n",
"469/468 [==============================] - 7s 15ms/step - loss: 0.5132 - acc: 0.9088 - val_loss: 0.5808 - val_acc: 0.9042\n",
"Epoch 14/20\n",
"469/468 [==============================] - 7s 15ms/step - loss: 0.5156 - acc: 0.9107 - val_loss: 0.6118 - val_acc: 0.9028\n",
"Epoch 15/20\n",
"469/468 [==============================] - 11s 23ms/step - loss: 0.5057 - acc: 0.9109 - val_loss: 0.6154 - val_acc: 0.9022\n",
"Epoch 16/20\n",
"469/468 [==============================] - 8s 17ms/step - loss: 0.5045 - acc: 0.9117 - val_loss: 0.6180 - val_acc: 0.9026\n",
"Epoch 17/20\n",
"469/468 [==============================] - 13s 27ms/step - loss: 0.5091 - acc: 0.9113 - val_loss: 0.6094 - val_acc: 0.9044\n",
"Epoch 18/20\n",
"469/468 [==============================] - 14s 29ms/step - loss: 0.5093 - acc: 0.9120 - val_loss: 0.5996 - val_acc: 0.9052\n",
"Epoch 19/20\n",
"469/468 [==============================] - 11s 24ms/step - loss: 0.5116 - acc: 0.9110 - val_loss: 0.6362 - val_acc: 0.9011\n",
"Epoch 20/20\n",
"469/468 [==============================] - 11s 23ms/step - loss: 0.4987 - acc: 0.9132 - val_loss: 0.6071 - val_acc: 0.9062\n"
]
},
{
"data": {
"text/plain": [
"<keras.callbacks.History at 0x137552c50>"
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),\n",
" steps_per_epoch=len(X_train) / batch_size, \n",
" epochs=nb_epoch, \n",
" validation_data=(X_test, y_test), \n",
" callbacks=callbacks)"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Two digit MNIST: binary logistic regression"
]
},
{
"cell_type": "code",
"execution_count": 119,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# the data, shuffled and split between train and test sets \n",
"(X_train, y_train), (X_test, y_test) = mnist.load_data()"
]
},
{
"cell_type": "code",
"execution_count": 120,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(60000, 28, 28)"
]
},
"execution_count": 120,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"X_train.shape"
]
},
{
"cell_type": "code",
"execution_count": 121,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"zeros_train = y_train == 0 \n",
"ones_train = y_train == 1\n",
"mask_train = [any(tup) for tup in zip(zeros_train, ones_train)]\n",
"\n",
"X_bin_train = X_train[mask_train]\n",
"y_bin_train = y_train[mask_train]\n",
"\n",
"zeros_test = y_test == 0 \n",
"ones_test = y_test == 1\n",
"mask_test = [any(tup) for tup in zip(zeros_test, ones_test)]\n",
"\n",
"X_bin_test = X_test[mask_test]\n",
"y_bin_test = y_test[mask_test]"
]
},
{
"cell_type": "code",
"execution_count": 122,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"X_bin_train = X_bin_train.reshape(X_bin_train.shape[0], 28, 28, 1)\n",
"X_bin_test = X_bin_test.reshape(X_bin_test.shape[0], 28, 28, 1)"
]
},
{
"cell_type": "code",
"execution_count": 123,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"input_dim = 784 #28*28 \n",
"X_train = X_train.astype('float32') \n",
"X_test = X_test.astype('float32') \n",
"X_train /= 255 \n",
"X_test /= 255"
]
},
{
"cell_type": "code",
"execution_count": 124,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# define data preparation\n",
"datagen = ImageDataGenerator(zca_whitening=True)"
]
},
{
"cell_type": "code",
"execution_count": 125,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"datagen.fit(X_bin_train)"
]
},
{
"cell_type": "code",
"execution_count": 126,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAVEAAAEICAYAAADm98d9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXm8XVV5/r+LMCQQFMIQQgiEIaABROZBVKYIogWsylAc\nECiitgWLfkCKFqW2ONEf/UhVaiGgglilggJWCChSkFEQQkiYJSQQJpmFIuv3xznPWe9Zd92Tc++5\n94zv88+9d6+19153vXvv9bzDet8QY8ThcDgco8MKnR6Aw+Fw9DL8I+pwOBwtwD+iDofD0QL8I+pw\nOBwtwD+iDofD0QL8I+pwOBwtwD+iDofD0QIG4iMaQpgSQvjvEMKLIYSHQwh/1ekxOVqHy7U/EUL4\nmxDCLSGEV0IIczs9nuVhxU4PoE04C3gVmAq8FbgshHBHjHF+Z4flaBEu1/7EEuCfgH2BSR0ey3IR\n+n3HUghhNeAZYKsY46LqsfOBJTHGkzo6OMeo4XLtf4QQ/gnYIMZ4RKfH0giDoM5vDrymF62KO4At\nOzQex9jA5eroCgzCR3Qy8Fx27Dlg9Q6MxTF2cLk6ugKD8BF9AXhDduyNwPMdGItj7OBydXQFBuEj\nughYMYQwyxzbBnDnQ2/D5eroCvT9RzTG+CJwMfClEMJqIYTdgQOA73V2ZI5W4HLtX4QQVgwhTAQm\nABNCCBNDCF0bSdT3H9EqPkklVGIZcAHwCQ+D6Qu4XPsTpwAvAycBH6r+fkpHR9QAfR/i5HA4HOOJ\nQWGiDofDMS7wj6jD4XC0gJY+oiGE/UIIC0MI94UQfJdIn8Dl2r9w2Y49Rm0TDSFMoBJmMgdYDNwM\nHBZjvHvshudoN1yu/QuX7figlbCBnYD7YowPAIQQfggcCAwrkMmTJ8cpU6a0cMvexiOPPPJkjHGd\nTo9jORixXCdNmhRXX31wNwo98cQTvSBXGKFsV1lllTh58uQ2Dq+78PTTTzcl11Y+otOBR8zfi4Gd\nG50wZcoUPvOZz7Rwy97Gcccd93Cnx9AERizX1VdfnQ984APjOqhuxre+9a1ekCuMULaTJ09m3333\nHfdBdSsuvPDCpuQ67gGsIYRjgGMA1lxzzfG+HSusUDHzWjPFiiuuqLEA8OKLL9baVlppJQBWXnll\nAP7v//5vyPmOobBybQdbee211wCYMGFC7ZhkJLnqZ6lN0PMB8Prrr4/PYHsYVq6rrrpqR8YgWetd\ntN+NV199FRgqu07KtRXH0qPADPP3BtVjdYgxnh1j3CHGuMMgqwY9hBHLddKkrk/56KhgubK1cp04\ncWJbB9eraOUjejMwK4SwcQhhZeBQ4NKxGZajg3C59i9ctuOAUavzMcbXQgh/A/wPlT2u57Sy5e7P\nf/4zkGi5VcN0TFTeqnS5uiZIha+OFYAXXngBgLXXXrvW9uijlYV4vfXWA+CZZ54B6lVUrcgvv/zy\nyP6pHsRYy1VykIplVS3JXLByzaE2K2+pezLFSA2EpIpKZiXzjORqz+tnjLVsh0MjmeeQDO37Klnr\n50svvVRrk/lN3wLJ3t5Dv1sVfzzRkk00xng5cPkYjcXRJXC59i9ctmOPjmdGERvRyqKVxtpj1Eer\njvraY1qtcgYCiYXIdrd06dJa25NPPgmkVW/dddetO8e26b42nEfMaNmyZXX3tyvrIENylBZgoTnO\nnXqQ5lV9xC4sW23ENnVM/TUOe37OZiwj1f1ztuyoR4nt5TKzc6j511yXNM/ciWhZptokOz0z9nux\n1lprAekZsOerv+47Fk4o3/bpcDgcLaDjdOmVV14B0urxpz/9CSivEFqZrDdYjE8MVivNBhtsUOtz\n7bXXAvC2t71tSNu73vWuuvN+9rOf1f0NlfhWSCvqKqusUmvTCqqxlWw0gwit9JKn5GMZh0LNxFLX\nWGONIdcR2/zjH/9Y9zfAY489BsCmm24KwLRp02ptTz31FJDs3//1X/8FwMyZM2t9dL98jJBYjJ4v\n/RwU++lwyH0XpZBAsUK9A7ZN70cOqyForsU2p06dWmvTu3zcccfVjWPRolRq68ILLwRg9913rxuP\nvWbOkhvZ45cHZ6IOh8PRAjrORMUstMJr1bI2RTG/p59+uu4cqA+cB1httdUAeOSRtDFjl112AWDr\nrbcG4De/+U2t7YknngASO/3rv/5rAObOnVvrk3uBxYogsZk8ps6OfxDZi1Z4MU8xAMvi3/CGSomk\nnNVAYoeCtA97/Pe//z2Q5Hv77bcPGYds3nvssQcAzz+fSjDpGdlwww2Berut5Jnb0AZdrnq/NC/S\nJO3zLxnlLB7q/RmQ3ndrNxVLVF87zw8++CCQ3rszzzwTgIMOOqjWR++07mvlqvFKnmKgrcjVmajD\n4XC0AP+IOhwORwvouDov5EHzzz77bO13qQq5igVDQyxE1++///7asZ122gmA3/3ud0C96rDbbrsB\n8IUvfAGAv/u7vwPgne98Z63Pl770JQAOOeQQoN6EIOovFUSqgxxlMBhB+jlyh5tUM6uO5zK3cpHp\nROrer371KwDe/va31/rMmlUp9HnPPfcAZTVMclmyZAkAW2yxRa1Nz9hdd90FwFvf+tZa23PP1Ze0\n13NmZdmKM6JXkTuSSg63XK7Wyar+eg70Tuu9haEqtg2233vvvQGYP7+yR0BOX+uUzDOK2fHIFKgx\n2W/JaOFM1OFwOFpAx5moVjA5DsRGStsutSJZ9qljMnirr8JfAB5//HEgrX5vfvOba23f+ta3gOSA\n0IpmVy8F4IuB2jANrZZa0bQyWufTWKx2vQYxlZyBWgdEHtBuHYZiJpLLF7/4RQDuuOOOWh+FL2nz\nhH0u3vKWtwCJUS5evBhIDAYSK11//fUBuPXWW2tts2fPBuodUfk9BhElJyCUWXm+Xdsi3+BQ2gQh\nx+Odd95Za9t4442BpGnOmFHJp6IQNqgPYRxuTHnQv2XCI5XxYD8RDofD0SLaykT//Oc/88ILL9Sx\nzDe+8Y3A0G161j4m25VYXymZhFYPJRB5xzveUWuTjUQrm5KOQGI/Crq/9NJKUpv3vOc9tT5iJRqH\nVj8YmhSlFLzbqbyM7USMsY69a64byVWrv8KeLBsQ4/zYxz4GwAUXXADU25o32WQTIAXJWwZy0UUX\nAcluqjAoy0TFZjbffHMAtt1221qbtvFqbGKkNiC/3xFCYJVVVqmTixhoHsJmn/d8K2cpuYjem3zr\nr+0jzU9yhhSq9otf/AKAffbZB0h2cUhsVeO2988TnozF5hhnog6Hw9EC/CPqcDgcLWC56nwI4Rzg\nvcCyGONW1WNTgIuAmcBDwMExxmeWd60JEyYMKSUhdU9qUyNDs/pa6i9zgFSIa665BoCDDz641ufe\ne++tu4dV6aTKyRGU03x7Xu5kgOQw0XilnnR7GYqxlGv13Lq/NVeah3w/NSTnoebemlnkKNQuJJkH\nrGp34403AvDe974XgLvvTvXWpNrr/rrO9OnTa33+8Ic/AGkXzGabbVZrU0iUxthLavxYyTbGyCuv\nvFInszy7ln6WQpSEUoiT3i+ZBUphizrPOiPl/JOz9/Of/zxQnyPY7rXPr51ngtP/U9px1SyaYaJz\ngf2yYycB82KMs4B51b8dvYW5uFz7FXNx2bYNy2WiMcZrQwgzs8MHAntUfz8P+BVw4mgGoBU+NyaX\n9s6XIHYqpiInkA1j0j3OOeccoN4BodAq7aOWc0FZgCCF0IiV2IBrrXJaNTXWbmSfFuMt13w117xY\nJ1teTE4bHiA5hnQdVR64/PKUT1jPjJisDYVRZif1kRNJzwcktisWZQuiSdYKsdK4Lcvq1lyjYy1b\n+yznOQTE4ErsreRM1PuSh7dZzU3vsrIxHXXUUbU2harJWSyNU5oDpPe9FBKp++t51P3tBprhqmUM\nh9HaRKfGGJXZ+DFg6nAdQwjHhBBuCSHcUkrM6+gqjEqug7gjqwfRlGytXEeq1g4qWg5xijHGEMKw\n9YVjjGcDZwNsuOGGEeqZpViAWIlWfLsyaLXIM2ZDWpG++93vAvCJT3wCSFs8IbEJMQ+LPNhWY9t+\n++1rfb73ve8BiZ3Y8etBy2089gEcLodiN2Mkcl133XUj1K/4+RbdPJsTpDlSGNK5555ba9NzoM0L\nCiuzTFJ2Vz0D1haWawQ634bCyJ6u5+q+++6rtc2bNw9Imkkpi1OvltVuJFsr17XWWquR/IH0ntoc\nv3n4k0VuUy310TU/97nPAWmzDKTNE4cffjgA3//+94H6d0zvdOnauq8W/txmDyPfHDNaJvp4CGEa\nQPXnslFex9FdcLn2L1y244TRMtFLgY8Cp1d/XjKSk21CAX31peqLTZTsoGKLYhCQmMaXv/xlIK1a\nc+bMqfURk9xoo42AepYou43Goe2iRx55ZK2P8k2WkiXk28hKNt0eQktyLVXyzL3a1j6mNrF+ywTF\nWPVcaBOEPPGQvPnbbLMNAFdddVXxPnZsNlGFArVVzcCam5ToRJs3JNeSNtMjGLVsS89yXtvIahil\nAHpBctA1xRYtk1UgvWRoWaI0k29+85tAeu+sPTvfim3HofvlQff2ezNSW/dymWgI4ULgBmCLEMLi\nEMJRVAQxJ4RwL7BP9W9HD8Hl2r9w2bYXzXjnDxumae8xHoujjXC59i9ctu1FW3XOGCOvv/56HfUX\njZaaJVXZGoWlfpeM0T/+8Y8B+PrXvw6kkhHKPwlJTZMqYOm9nE5SGa6//noAzjjjjFqfb3zjG0C9\nyiHkx+R8sirBSEMmehH53nnNi/INaM+7zROr4HgF0Fu5KJhaspN6bedVzqfPfvazAGy33Xa1ttxh\nKbno+YD07PzVX/0VkMpKQCpqePHFFwMpLC4vR9PP0PtqHYaaT6nRjUoe585ae36e+9VmPfvABz5Q\n1/fhhx+ute28884AnHRSJcz1+OOPB+qdT7ljyd4rz4Oq56mVyCHf9ulwOBwtoK1MNITACiusUMfe\n8qD00uqVFzmzoSViNgq0FmOxuSFVJlfnWceSGKhKsV522WUAXH311bU++++/P5Ay+9jtn1rJ8nyo\n3R5sP9bI2bZkpnlRkUELbdsUu7BzJlktWLAASFsyLZMVE5V8S6EpkoeeKzkXITkRxS7tMyfGmjvG\nLKvpUedh09D7apFv6cxLS0OaozyLva4JaV4VnmaLR6rCwHXXXQfUZ+468cTK/gA5IaVJlvKZlo6N\nx/vpTNThcDhaQMeX0twmoRASazfNS69qSyCkfJNakRS4ffPNN9f6KMu52I1lLLqfVr8rr7wSSGEz\nAB/+8IeB+u1nghiO7HkK3+qlhBXjiUbJW8RYtN3Thr5JLno+9DxYxqPf1WafGckjZ4tKNgJw+ukV\nB/VPf/pToJ65fOhDHwLgrLPOApKcB7GukkW+XTPfZAJJdpJnozlbtGgRkHKAQmKZ2upr2b/ud/TR\nRwNDcw1DYpml3L65TTf/f0YDZ6IOh8PRAvwj6nA4HC2gI+q8dULIsCx1vpR5JS+B+rd/+7e137fc\ncksgZXFRwSo5KyBRfqmWckRACoX6zne+A6Q9+N/+9rdrfT7+8Y8DifKXwmx0TGO1qukghDjlkEol\n9UsOPJsdS/OinJClhBfaKy01zO5eydX5UoYlqZbK0qWidJCcGTK9KPMTJNPNXnvtBaTncxBlaVVl\nzbVUbM2vfV/lbNK82vdFclRIo97NPffcs9bnpptuAtIzpHAzSM5dhUTpnbYmutJ9hTw0aywcTM5E\nHQ6HowV0hIlap4tWtnz1sCuE2Myvf/1rAB544IFa27HHHguk1XKdddYB6h0Z+R5e63TafffdgbR/\nWkHVd911V61PHjxcCsVRKM8gMxbrANBKrzkTO7HhaSo6pjmz5+t5UA4E7ZO3cy/mWipxK4akqgZb\nb701UB+UreB65WIQ64VU8UChWWIwJSdFv8OGleWaheRh0yHmxd8sS1SuULHNgw46CKgvha18Baee\neioAJ598cq1N2bjygnlWFvm7V2LS6jMW76kzUYfD4WgBHWGiNhRF0Gqh1cvWYspzds6dO7fWpvyQ\nCpbXdkHLTmSflL1SWZnsfdRHttEPfvCDtT55xhq79U/sSWPU36VM3f2OUvC5bFf5pgRILE92Zcsy\n85K6QklDUV8xGEjsVltKVYNHNjVI7EkMVOFxkJisxqZ7lOoF9TusHVpMPLcl2uxWmle9CzZzVl72\nPK9NBslnoa2ddhOE3sVc+7Dn674aY0lDyrOvWbY8Urk6E3U4HI4W0PalNMZY5zUTmxCT04pgVy9V\nZVTCCtVZgVS9cbfddgNSsgKb0EDXVOZ0JSuBtMrpHrKd2dVIK2tpS2qeSEE2lpIdpt9RSrqieZAW\nYOUiW6Tm0DIe9dNzITu4zWwvmcl+ar24YhoK3JbNzTLh973vfQBccMEFdX0gsaHcRm/t+d1aY2ms\noAQkVvOSVpUzwkZsz7I85fZVVV4l+rEJQE477TQAvvKVrwD1rDdnjrn91ULPoPWJ6F3U+fppbd35\nO708NJNPdEYI4ZoQwt0hhPkhhOOqx6eEEK4MIdxb/bnm8q7l6B64XPsTLtf2oxl1/jXghBjjbGAX\n4FMhhNl4CdZeh8u1P+FybTOaScq8FFha/f35EMICYDqjLMEaQqgLK8hVIql9ll5LbZNDSZl9IBn+\n//Vf/xWAHXfcEUhqPiQ1UU4jhcvY+yi35BFHHFF3T0hGa6ly1oid75HPnR3QnQ6IsZbrcMhDzuzG\nCTkjttpqKwAuuSRVrNAcq01B8trnDilAW2YBW4QuLyyncVhzwm9/+1sgydA6luRs0rjlsLSlYbpR\nnR9LuSqLk/53GBpSVMrZmTt9bLjgFVdcASS1Xg7d97znPbU+2nyha9sNEnlu4ZLqnW/CKPWxpiOo\nN9EpD0ezGJFjqVrLelvgRkZRgtVLJncnWpWrl0zuTrQqVy+Z3ByapkghhMnAT4DjY4zPWTbZbAnW\nGTNmxD//+c91K1Vu/C2t7nIGiB3aQHoZlGWoVoanhx56qNZHeSu1It1///21thtuuAGAww6rVFRQ\nVhmFxEBiJaXMTLq/VjYxW/th6WbH0ljIdZ111omvv/56nfYgNtBo84HCjc477zygvrig2Mv8+fOB\nlFNyv/32q/XRFlKbkd78X8DQUBYFeUMK9heTtSFSYpySq56B0rPbjRgLuU6ZMiXGGIvlv8UES8Hu\nmjPlaz3kkENqbZKr3k9pE6ooAbB0aeVbr63bjXK45uOw/UtOLzHOvKyz/R9LRTIboSkmGkJYiYpA\nfhBjvLh62Euw9jhcrv0Jl2t7sVwmGiqf6v8EFsQYzzBNLZXXFfJgaq0M1vakcqiyZcq+BSn0RZnt\nlUBEQdaQGMf5558PpGQUAAceeCCQWI1srHarW56swK5aCtbPTRXdHgoz3nLNQ1DExu1caAufbNy/\n+93vam1bbLEFkOQrO5W1V4lhyM4q2dlrq0KBQuaUUASSPEuha5Kffkrmlm2Xtpt2GuMtV81DqVSy\noLleuHAhkBL4QLKJ6h3WfNpy16qxJI2zFBIp5Jt0YGhoVSkxTc6g7fkj1TCaUeffBnwYuDOEcHv1\n2MlUhPGjajnWh4GDR3RnR6fhcu1PuFzbjGa889cBw32avQRrj8Ll2p9wubYfbS9UN1ypANFpqcXW\nMaMyAfPmzQPg4IPTIvrud78bSM6id77znQD88pe/rPVR2VuFy+yzzz61NqkMUgFKeSvz3JQ224/G\nKbVeKqEt8ypzRL+iVNAsL7sg+Vr5SxWU6cVeQ/O30047AckEY3e/qKSEVETllrXH3vSmNwFJDbV5\nXvOMRFadz8uBNHIq9ivycERIMspzCZTKiWt+7U4yqe3vfe97ATjyyCOB9B7ba+qnfS70POUmNutw\nylV0K7t8N5Wei1bCELvPqONwOBw9hK6LArfZmwSxvb33rmgjNqhavys8pRSgq33QWnVsKItWKTEO\nrZ6W8eiaYjFTpkyptWlfcZ49v9/ZZ7PI2YRlb5p7zXWptK7K5uaOHkiOKGWkt9qLAufzcsjWSZGX\n9LWZiCRrOatKDs9uDnFqF/JNDZBCzg444ACgPiOanE7a+CJGajW/PO7Yagg5E9WzYx1ccjTqWbFM\nNs/IXwraH6lcnYk6HA5HC+g6JipYxiCWqBXOhkWJOSoYO2c3kFYWHbOMQyuoVr9S2IrOL61aIw3M\nHVRoDkvZcvLytZDkr1AYyckyQZ0vuVh7p54DnVcKd8ntYTZbkfrZa+ZjdCTYuRP0vtrttHn9pTxX\nLwy1NVuZ5/lH9b7a0CfbH8rbPvPxtiJXZ6IOh8PRArqWiVrGMJxHH9Kq1cgLnNfisatOM6uWY3yQ\ns377t34vVVgVxCDVVvLClhiwo70o1c6S7CSX0j793F8BSdbd9J46E3U4HI4W4B9Rh8PhaAFdq86P\nFM0Ey0rV7+asSo6Rw1X23kMz6ngjU043oW8+osPh2muv5aabbmLJkiVsv/32HH744Z0ekmOMcOed\nd7Jw4UKeeuopZs2aVZdcxNG7eOWVV7jppptYunQpq6yyCttss00t1rsbEfIsSuN6sxCeAF4ERpY6\nujWo4t0bqJgvHhrlddam9XFvFGNcZ/ndegsdkiuMjWxdrsOgg3LdmMr+/4eAVYHNgHuAkWSJbptc\n2/oRBQgh3BJj3KGtN63c95+ADWKMR4zy/I6Mu1fQyflpRbYu18Zo9/yEEFYDngG2ijEuqh47H1gS\nY2y6LlQ7x+2OJYfD0U3YHHhNH9Aq7gC2HKZ/x+EfUYfD0U2YDDyXHXsOWL3QtyvQiY/o2R2451ig\nV8fdLvTq/PTquNuFds/PC1Rs3BZvBJ4v9G2Eto277R/RaiGsnkOvjrtd6NX56dVxtwsdmJ9FwIoh\nhFnm2DbA/JFcpJ3j7nt1PoSwYghhIjABmBBCmBhC6PvQrkGAy7b/EGN8EbgY+FIIYbUQwu7AAcD3\nOjuy4dH3H1HgFOBl4CTgQ9XfT+noiBxjBZdtf+KTwCQqFUkvAD4RYxwRE20n2h0nuh9wJhXm8N0Y\n4+ltu3mTCCHMAM4HpgIRODvGeGYIYQpwETCTSvzawTHGZ4a7ziDB5dqfcLk2OYZ2fURDCBOo2Dvm\nAIuBm4HDYox3t2UATaJak3tajPG2EMLqwK3AQcARwNMxxtNDCCcBa8YYT+zgULsCLtf+hMu1ebRT\nnd8JuC/G+ECM8VXgh8CBbbx/U4gxLo0x3lb9/XlgATCdyljPq3Y7j4qgHC7XfoXLtUm09BENIewX\nQlgYQriv+rVvhOnAI+bvxdVjXYsQwkxgW+BGYGqMcWm16TEq6kNfwuXavxiBbF2uTWLUH9Eq3T8L\neDcwGzgshDB7rAbWaYQQJgM/AY6PMdYF/8aKDaS9+2XbBJdrf8oV+lu2nZTrqG2iIYRdgVNjjPtW\n//4cQIzxX4brP2nSpOtV7W8QsXTp0ie7PVHFaOQ6ceLE61VhcRDxxBNPdL1cYWSyDSHsuuqqqw70\n+7pkyZKm5NpKTF2J7u+cdwohHAMcA5ViUkcffXQLt+xtnHbaaQ93egxNYMRyXXHFFTn44IPbM7ou\nxFlnndULcoUmZGvlutJKK3Hssce2b3Rdhi984QtNyXXcA5OrOwfOBlh//fXHXVXKa4kDTJo0CUhJ\ne59//vkh/fN6P7bioFd4HAor13XXXXfc5SrZ2QS9+l0VI23FR8lVCbhLVVxLxwYdVq7Tp08fd7mq\nxpmtmquEzWpbc801a215PS3Jvtm68Xnl3kZ9mkUrT9GjwAzz9wbVY47ehsu1f+GyHQe08hG9GZgV\nQtg4hLAycChw6dgMy9FBuFz7Fy7bccCo1fkY42shhL8B/ofKjoZzxmtrlqh3MzRbfSdOnFg79uyz\nzwJJJbS1eKQGSE1Qm72XrtVNZVrHC2MtV6nMkktJjZLppJHKrfOsmWWttdYC4KWXXgLgxRdfrLVJ\n3ZPDS7KT+gdlE0E/Yyxlq7mTPKxc9e5oru37lpfCVputezZ1aiUa6Y9//OOQ83XNvES6latkveqq\nqw4ZdzOO9JE621uyicYYLwcub+Uaju6Dy7V/4bIde3RNxpucZdqVJXf+WKeRWKJWq5dffhmoN1Sr\nj5iKWIrtp3uUmKig83QPe56jHnkFTjEHO69a8UsMP5eDnAx2vuUg1HNhHUtiNpJVSZvIGah9ZsRw\ndF+NZ9CdUZKH5vrJJytljKZNm1br8/jjjwOJUUoThKQ1SI7SJp55Jm1rf+65SpinZPbUU0/V2uQk\n1vdB49F1IMlOP1944YVa22qrrQbUf0NaxWA/EQ6Hw9EiuoZGiTlohbOsIrd/WHtnjlLt8d/+9rcA\nXHXVVQCcdFLa7aZVc7PNNgOSHcbeP7fJltjIoNnXlgfNUc4YLBMUG5k8efKQttxWrb4ltvnYY48B\n9bKfMmUKkJiHrmNlJ7nq2vb+6id22simO4gQu9Q8We1Mdmi9S3bOFLyvedXzYW2iy5YtA5Jcv/CF\nL9Ta7rjjDgC23LJScunqq6+u+wnwnve8B4BHH60EHtgQKbFSq43acYwGzkQdDoejBXSciea2Jrsi\nCbm90rI9MRyd94Y3VMqz3HXXXbU+n/rUpwA444wzANhpp51qbf/xH/8BpJUtt6dAWklL3n0hZ06W\n8QyiHU3MRLLTHFoGkNtCrb1UcyybpM63c/nAAw8AsPfeewP1drn//d//BeC6664DYM899wSSDQ+S\nRpOzZqiXv4W1m7bCXnoVer6lPWieZKuExED1LlqWKluk2OGPf/xjAI488shan+uvvx6AY445BoBz\nzjmn1qbnSSxzk002AeDf/u3fan1OPvlkAHbccUeg/pui50/j1ljXWSft7hypvXTw3m6Hw+EYQ/hH\n1OFwOFpAx9X5PFi3UYhRo2D33PBv1TEZqqW+Pfjgg0Pun6voJQdCo2M6X+qCVQkGUZ3Pw9I0z43k\nWprffO7svCqYWuf9+te/rrVtvfXWQJLHkiVLAFh77bWHXEsmAzu2fJwl9X4QcyrkZjfJeenSpbU+\nUtUVtmQzfOn3efPmAfDVr34VgPvuu6/WZ/bsSnY+hUMdfvjhtTa9u2qbP7+yV8CaE55++mkgqeo2\n/EljkswdjDl8AAAgAElEQVQ1HvtctXPvvMPhcAw8Os5EZZzXCpcbfu2xUviQ+uVZe2T4hqFB3TNn\nzqy1zZgxo+68EvKVqeTYysOwPAi/gnybnxgADA0zsdqAnBG5hmAzcClcRizIXk/Mc7311qv72zo5\n8mvbZ06hVDqm50kMyP5vgwS9r/n7plAySOFPmjM755pPhaCJgVotYs6cOUAKRVRfoJZy8Te/+U3d\n37/73e9qfc4999y6863DUXLVe6tx2+dypGFsg/cUOBwOxxiirXQpxshrr71Wx9KGCzOxrECrXSn3\noKD+Nhhb0P207VOhMQCLFy8GYNasWXXXtuErOla6h9it7pEHlw8CQgisvPLKRTaf25zt3EkejTYq\nqL/mWfZtgK997WsA/PM//zOQtApINq499tgDSKFO1j6Wb6Kwz2WuIVmmkv9v/YoYI6+++mqdzN74\nxjcCSR6l5zyXoz1fttBf/vKXAHz2s58F4P3vf3+tzz/8wz8AKcTJah8KbdJ3Q/eyTPiiiy6qO8+G\ntYnV6vmQfFtJLuRM1OFwOFqAf0QdDoejBSxXnQ8hnAO8F1gWY9yqemwKcBEwE3gIODjG+Mxw1zDX\nYsUVVyyqQVKX9NPmAhTlLql9eehMabdBft6b3vSmWptCHkT91deqdrnTyBqem8mP2I1OprGUq9S+\nklxzR1vJsSPV2TqGcqeTdr/cdttttWPKiaBsQVbt23nnSumgd7/73QB8+tOfBupzjjZyGObjz5/B\n4fp3A8ZKtjLT2P8zV+P1vlp1One42WJ3t956KwDf/OY3gSS7a6+9ttZnl112AdKzYp8rPQf6Plx2\n2WUAfOQjH6n1UY6Ms846C4DPfe5ztbbtt98eSGaJhx+ulFGy+ThGKtdmmOhcYL/s2EnAvBjjLGBe\n9W9Hb2EuLtd+xVxctm3DcilSjPHaEMLM7PCBwB7V388DfgWc2OxNS0Xg8uBsyxJy541lA8PlpLSO\nAAXibrTRRkDjlabEinLGYv8eLojcrmzd6IAYD7la5PNSmifJWvIpbVDIg7s///nP1/osXLgQKOcr\neOKJJ4CUL0F76adPn17r06jQXa5Z6Pkq7cPuNoy1bO3zq+c6LxCnHKAw1Bl400031dquuOIKAH7+\n858D8Bd/8RdAvVz333//umvaedZ9xST1zNjQM93/zjvvBGCfffaptUnzVI5SORqtFjNSjNYmOjXG\nqC0KjwFTh+sYQjgmhHBLCOEW+486uhKjkqtV0R1di6Zka+VqTR+O4dGysS7GGEMIw0anlkoml1aW\nPBTG2h31u1Yda2NUWx4cbQN0ZRP58pe/DNRn2hZjEUvVPezqm+e0tIwntxHpb/s/diMTXR5GIleV\nTLbzkttCS7LL58WGruXzqaBsZeaBFECfVzWw11awvfJIWg1BHwndtxHL1DPQaFNGr6CRbEslk63M\n9O7kG2CsdijZaUulNAaARYsWAel9veaaawDYbrvtan0kx1IlC0Fy0DZevb+QAu+17VN2WEjPg64t\nZlrKINYsRstEHw8hTKvefBqwbDn9Hb0Bl2v/wmU7ThgtE70U+ChwevXnJSM52TIQrQha7RrZzoRS\nooicuditnQ899BAAP/3pT4HksYXEcNZff30grZDWbqpVV9cuZWfPGagN1u+hwPuW5GqRb56QN9XK\nPs/HaeWs+RdzlMZhE13kTKWUtV73FSsqaRg6VqqxJKjNspSxrNPTBoxatiUNQ0wwt5HC0NpXW221\nVa3t9ttvB+DQQw8F4OMf/zgAH/rQh2p9dG2dbyN1ZBLUe6b39uabb671kb1UrNmaJfRclDZPjBbL\nZaIhhAuBG4AtQgiLQwhHURHEnBDCvcA+1b8dPQSXa//CZdteNOOdP2yYpr3HeCyONsLl2r9w2bYX\nHdk7b5FnbRLdtup0ruqXsqzkhe5sJMDPfvYzIIU1KN8gpL27CtwuqWh5CI29v9QC/R95ftHhxttv\neP3114sG+VzltkHZmhfJrmT2aCTz3NFnQ5Skvkt2Kv9g1bi8dElpE0cp5E6wamY/wzrZ9F7J+bPu\nuusC9Q47yVPmM/0EOPXUUwHYeOONgRR+ZB12w5WWscf0PElmyoEBsM022wCwYMECIDmzIAX+6zlU\neJ29v2dxcjgcjjairUxU2z4txExzllfaXlcKf8odOfpbrBNS5hcF79prn3baaUAqs1pybOUOCIs8\ne1MeapVfq1+xwgorFLfDCjmDgCSHvCwxDM12X3Jg5IH8pftrO6CKFdrwtlzDsWPOHYa6Vym7V7+j\nFPqVV3CwfZRHdL/9KpumLrkk+bC02eHMM88EkhPIzmv+npc2QUijUF8F1gMccMABQHrvtcUU4Lvf\n/S6QtpuKWVsm7YXqHA6Ho43oeGaMPJFByQ4y3BZCGBoIrza7jUuB1jpf5XMBNthgAyDZTbQilTLr\nl3KFiqnktaEGgX02g9ymaLWAPHlLo/NLtsk8NEnB1ZBYkMpjy85WCp0rMdE8ZC1/PgcVej/E6DVn\nSgwCKX/nV77yFQBOOeWUWtsWW2xR16bQppLmJ3tlKURJDFJM1spOAfSye/7hD3+otan+k1CyiY4U\nzkQdDoejBfhH1OFwOFpAR3QTa8QVLZf6JCNyI0OzVQmH20FhQ6mkyon6X3jhhbU2GaSl4sv5ZNX5\nRuWU89CmXL3Pr9XPsCqRwn80H1LRSmqbYFXlXNXP8ydAekakWtqyuVLl8v3x9rnKi65ZuWr8eeHD\nQUzKYZ/fPFStNB96h7S7zGZo2nfffYG0a7C020zvVL5zCdLzZL8hkPKKQjItbLvttgD8/d//fa1N\nO6X0fsoE1IqZxpmow+FwtICOMNFSCEG+t9oG+OYsxjIYsYc88NquVAquV8D1hhtuWGtTZuucgZYC\nx0vOEfWX46FUKG9QnEylvef5nDVy3jRi7CXmo7nWT5vTcvfddwdSvkjdv+TYyvPVwtBntFSYrVsz\n248n9L5J1pKd3dwip66C3fXeQQp2zzW2Uuha6V2WXCRzPQ8qnQypcsWBBx4I1L/vv//974GhYYul\n/ADNwpmow+FwtICOMNFGQdl5JnF7rBT0rhVJTFCrlmUSyi2q82fPnl1ru/fee+vuVwr8zlmV3Tqo\nsclmpnAqy6Tz/3EQoDnLmX3J1lzadplrAmI6lgnqmGRtw1c+85nPACl/ZK6xWEjmpexcuc3dsm0b\nuN/PKD2/0u40H7aOkpioMixZW/nZZ58NpG2aCr6372uey7WkuUr299xzDwCHHHJIre2uu+4C4Pvf\n/z5QXyJ90003BVIeYWk49rnybZ8Oh8PRRnRN5HBumyhlQNcxu7Ll9lL9tLbJZcuW1Z1vmag892Kr\npVUot+eVbKJafcVILfohG3ozsExSXk8xFM1rySaqNssGxD7kcc+9sZBYkObcynXevHl1bbK5WVnk\nWkxp26fGUbL9lcbUj7AMPd/goHmVBgbpXfrHf/xHoN4medRRRwHwne98p+56di7zZ8UmelH+X23r\nPuGEE4BUswkS2/z2t78NwOGHHz7kf5HWonvZ936kPoxm8onOCCFcE0K4O4QwP4RwXPX4lBDClSGE\ne6s/11zetRzdA5drf8Ll2n40o86/BpwQY5wN7AJ8KoQwGy/B2utwufYnXK5tRjNJmZcCS6u/Px9C\nWABMZ4zK6+bqc4leC3kuUhi6d15/2yxOBx10EADXX389UJ9P9O677wZSfsPSHmupHLq2NRXkKn7e\nt1sxHnK1DoFcjc+dgzBUnS5dS+c99thjQMoTCkklK4UoKeBa6nhpE4QgddGGT+XOjLyUCXSnw3C8\n39c8LKzkCFafN7/5zUC9zJSjQuaWo48+uu46MDQT2q9+9ata2957V/JK33bbbUDKA7zlllvW+ijw\nXtmcbE4FQaFvKnZoHYYjDbwf0VNQrWW9LXAjoyjB6iWTuxOtytVLJncnWpXrIO7OGg2a/uSGECYD\nPwGOjzE+l4UENFWCdf3114/DGW1zplJigo2y7eRtYi6QyqleccUVAGy++ea1Nhm0xUbkILIrk5wR\npSJ0gs3YbseTj7fbMBZynTp1alxhhRUaFiDMt3/a3/MigxYKRVEGdG3lA/jRj34EJEZa0hB0zVIG\nLqHk3FB//WwUItWNGAu5Tp8+PcYY6/7nUl5VqNdCFNr04IMPAmnjA6TyyWeddRYARxxxBAC77rpr\nrY9kLgeVlblkpOD6c889F6h/3/bYYw8ghVFZZ6/+F21J1XfHEjy7fbgZNPV2hxBWoiKQH8QYL64e\n9hKsPQ6Xa3/C5dpeLHdZDZWl5z+BBTHGM0zTmJXXhaH5Gkvb60r5J3Omo1XEbgHUSvTWt74VSGVa\nAXbYYQcgrUSlvJW6RylgPE88UgqxKjHXTmO85TpcDtYSE8zlC2mOxWquvfbauusBTJs2DUg2LyWY\ngWTTzJmjtbXnCS4sE5b8GtlSu3E7b7ve19xmbW3I2uYpze/rX/96rU2VJC6+uPJtV4Z7C5mHlJfU\nzrPuq3ykO+64I1Cfz1RaqLLX229BHjKX5wGGkQfbN6ObvA34MHBnCOH26rGTqQjjR9VyrA8DBw9z\nvqM74XLtT7hc24xmvPPXAcMtuV6CtUfhcu1PuFzbj7ZbyYejyqLZUp9KIROlDEtS73S+aLmcSZBC\nJI499lgAPv3pT9faZKBWSFRJRcudDFbtzNV/jbHRXuB+Q4xxSOabfM5KWZSkxjcqQqdjeUFDGFo+\nwu6ayZ+VktqW790v7cuXg1FmARuONQilsPP3Ic/iJJOINZvIaaPSH8qmBHD88ccDaR4V8mQjAVSG\nWY4lZV4CuPHGGwHYf//9gfSe2QgRyfWhhx4CktkHklzzwpLW/FZycDZC97qNHQ6HowfQNfEaecnj\nkvMmz3gNQx0AYiN77bVXrc/8+fOBlE37i1/8Yq1NK+BwWYfsfYUSW9XKWiqcNSiZ7UvI5Vmau9K+\nevVX6JjCXmy4i85T+JOcDABvectbgMRmSvlehwuPy/tBCn0r5YsdRGg+SyGJ0hCk3c2ZM6fWphyj\nKmecM32A+++/H0jysCFHCk8U8yy9WwpX1Dtsvxd5ieRSue2Rwpmow+FwtICuYaJCo3Cg0rbLnKVq\nhbQlkxVyoe1ndmXKmVKeRxKaC2XRmHz3ThliFXbFz8PTLBPMsy7JBjZ37txaH4Wu/eAHPwDgbW97\nW60tfy50vRJzabQZQtcZlIxNI0Wj0C/Jt1Q7a8aMGXVtVi6578Ger/da8igx2UZhafmmmLGAM1GH\nw+FoAV3HRAXL/obzEFqIsahviRGW2IT69cp2vl6HlV3JFirk+SqVRV6Z6i1kAyttwihVk3SML/Jo\nh9JWX/VplNildL5kXaqG0Cn4k+VwOBwtwD+iDofD0QL6Rod1da0/0Yxc1cefgd5BM87aXtnM0Dcf\n0UZ4+eWX+dnPfsYDDzzAqquuyp577snWW2/d6WE5WsSf/vQnrr76ah555BEmTpzIrrvuWpfm0NGb\neOmll7jkkku47777WHXVVZkzZ04t7rcbEdr5tQ8hPAG8CDzZtptWsDGV/cQPAasCmwH3AENrsQ6P\ntWl93BvFGNdp8RpdB5ery3WM0VNybetHFCCEcEuMcYc23m814Blgqxjjouqx84ElMcam68y0e9y9\nBpdrf8LlunwMghFpc+A1CaSKO4Ath+nv6A24XPsTPSfXQfiITgaey449B6xe6OvoHbhc+xM9J9dO\nfETPbvP9XgDekB17I/B8oW8jtHvcvQaXa3/C5boctN0m2m4YG8uWMcZ7q8e+Bzw6EhuLo7vgcu1P\n9KJc+/4jChBC+CEQgaOplJC9DNgtxji/owNztASXa3+i1+Q6CDZRgE8Ck6hUOLwA+ES3CsQxIrhc\n+xM9Jde2fkRDCPuFEBaGEO4LIbSNmscYn44xHhRjXC3GuGGM8YIGY5wRQrgmhHB3CGF+COG46vEp\nIYQrQwj3Vn+u2a7xdztcrv0Jl2tzaJs6H0KYACwC5gCLgZuBw2KMd7dlAE2iWpN7WozxthDC6sCt\nwEHAEcDTMcbTqw/UmjHGEzs41K6Ay7U/4XJtHu1kojsB98UYH4gxvgr8EDhwOee0HTHGpTHG26q/\nPw8sAKZTGet51W7nURGUw+Xar3C5NomWPqIjpPvTgUfM34urx7oWIYSZVAzbNwJTY4xLq02PAVM7\nNKxxh8u1fzEC2bpcm8SoP6JVun8W8G5gNnBYCGH2WA2s0wghTAZ+AhwfY6wL/o0VG0hfhjW4XPtT\nrtDfsu2kXEdtEw0h7AqcGmPct/r35wBijP8yXP8JEyZc3+812BvhT3/605PdnqhiNHKdOHHi9apy\nOoh4/PHHu16uMDLZhhB2XWmlla5XJvlBxLPPPtuUXFtJhVei+zvnnUIIxwDHQCXf46abbtrCLXsb\n8+fPf7jTY2gCI5bryiuvzEc+8pH2jK4L8bWvfa0X5ApNyNbKdcUVV+Ttb397+0bXZfj5z3/elFzH\nPZ9ojPFsqluwJk2aNO6qkph1qb543gdSjZZGNZYGYUPCSGHlut566437BCnhsq3UKpmpqqOVuaq+\n5hVFSzWehFKi4EGTvZXrGmusMe7/fF6NFZIcGiVuzmuq2YTczSR8Hku04lh6FJhh/t6geszR23C5\n9i9ctuOAVj6iNwOzQggbhxBWBg4FLh2bYTk6CJdr/8JlOw4YtTofY3wthPA3wP8AE4BzxmtrllQx\nqW2N6LrarNonlX3SpEkAvP7667U2lVF+8cUXgaGldu21Vl+9a7NxjRnGWq65rKx6rHmVPFQm2Z6n\nnzrPlr3OVXVbJvsNb6gkAnrhhReApN6Xyu/q/v2uuo+lbPOyyBaa48mTJwNp7qH+3YOkxtv3VSp+\nyQSjb4CeFfV56aWXan30jKjveKMlm2iM8XLg8jEai6NL4HLtX7hsxx4dL1TXzOqvFSVnHpBWtpyl\nWsah87RaWSP0E088AcA661QiGdZbbz2gnhVdemlF41EYz4Ybbjjk2mKuuna7jdvdBslVchEjXHPN\ntIVZDGW11VYD6udM5z322GMA3H///QDMnDmz1udNb3oTAMuWLQOSNgGJ2UgeukdJQxGsc1HyVx8x\nppxJDRry90us0b7HcvDp3Zo2bVqt7cILL6zrf+aZZwJJPgCnnHIKAIcccggAzzzzTK3t3nvvBdL7\nqm+BPT8fq5WZnoex1DoGJYuTw+FwjAs6zkS1omnl16qRhyVBYhHW1qFVT+fpp2Wbuod+2lCYjTba\nqO48sVVrxznggAPqfl511VW1NjEjna/VrzT+QUJuh86ZCyR29/DDlXC8HXfcsdZ24omVXBGS+V13\n3TWkz8c+9jEgsdVPfepTtbZFiyolehYuXAgkucpOB0lG0iJsm/qLsYjpWLY7iMhDikqMLreJSj4A\nX/3qVwH48Ic/DKT3XtoEwHbbbQckBiotBmCzzTYDYPHixQCsscYadfe0Y2pkt1XbWDBTZ6IOh8PR\nAjrORMUqxQpkl7Irg9hLzvYgMUYdE8u0K1NuU50yZUqt7bnnnqu7n+717LPP1vrI63vTTTcB9XY9\nMZZBt5XlyO3XsmFZViEPurypv/3tb2ttp59+OpAYx7x58wD46U9/WutzwgknADBr1iwgsU6AO+64\nA4APfOADAHzve98D6qMurK0O6mWuMeU2UasF5TbVQYD+5zxqQhoHpHdKst5hh1S5+IYbbqi7nthm\nybuu985qdc8/Xym1tO666wLw9NNP151jIZZZejdzBmq/KSNlpc5EHQ6HowX4R9ThcDhaQMfVedFo\nqX9Sz63aZJ1EOdRPlF3qhlXb1EfhS0899dSQ+8vA/cc//hGoD6uQiUAhTjb8Kaf+jfZ4DyLkiJHq\nbudFWGuttQCYO3du7dhFF10EwBe/+EUghctYx85RRx0FwL//+78DcNhhh9XaJBeF1Eit19+Q5Kln\nwIbJ5A7Pktmm0XPZr8g3QWg+pGZDmjOp2FLvIc2jngfJtRSipPktvT96v2Va03tr7ysZWpnpW5A7\nllrB4D0FDofDMYboGoqUG+ntCpGHC1kjsPqJLcqYba8np4bCKOzKlodUyblg2aqM2DKe24wzebB/\nKQxrEBmLoLlaurSSZHz69JQcXY4DhcJY9n/llVcCSdbaoqmQFoCvfe1rAOyxxx5APZM9/vjjAXjo\noYeA5BTcdddda320iWKvvfYC6tmU7ivNSFt+rYZjw+AGBXn4kN6btddeu9bn2muvBeAd73gHUGaJ\nYqBWqxP0zCxZsmRIm95l3V/vu72/Qqr0XJXev+EY9WgwuG+3w+FwjAHazkRjjA23RJZWjZztlUJL\n8i2dG2ywQa1NDEdtNlu3ztM1xZS0pdC23XjjjUA9G1K4VCnIPz9/ECFbl35a+5ieg8svr2zltkxS\nc53bUDfffPPa7+9617vqrm3DZL7xjW8ASYvQRgltHwXYfffd6+5lZZ4nshHrtBs1BsHWHUIobukU\nAxWT1BxCmutHHrH5nyuQNifbdmnbqI7lW28hsdoZMyoZ/fRuW7ard1KM1Npb9Xspuclo4UzU4XA4\nWoB/RB0Oh6MFLFcfCSGcA7wXWBZj3Kp6bApwETATeAg4OMb4zHDXyK5X93dOp/NQp9I51mkjQ7/U\naRmTtZMBkoqtcAg5OSCph7qOVAfr5FDb1ltvDdSH2eQG6VzdKf2P3YCxliuUTRmaKzkUrCNBzoBb\nb70VqDfBaMeR1K8tttgCqN8ff+ihhwLJGag8CJDC2dR27rnnAskEAEN3IdldL3IySbUslazo1l1q\nYynb/NnNHUqaMysXyS4PO4QkT71Tml9rGtE9SjsU9fzIYajwOCsLfTs23nhjoN4R/OSTT9aNYyxk\n2AwTnQvslx07CZgXY5wFzKv+7egtzMXl2q+Yi8u2bVguE40xXhtCmJkdPhDYo/r7ecCvgBNHM4B8\nJcj3sMPQPbClMBOxEIUmWVYkR5CyBdm222+/HUhsJM9xCSlzjAzlm2yySa1N+71l6FbYTreXhh4P\nuZbygcqRIMZi987feeedABx33HEA/Pd//3etTc4BMQ3N/c47p+KUeh507VIGdMlFDi2bN+HBBx8E\n4NRTTwWSowngk5/8JJAcFtpnbzUMy3C6CeP5zopV6h3S/FxyySW1PnkYknXG6bw8+5KVXSN2mG+M\n0LeglA9D3wJbzltOZb2fOr+V/L+jtYlOjTFKJ34MmDpcxxDCMSGEW0IItwyyl7pHMCq52rIcjq5F\nU7K1ci3tLnMMRcsxGjHGGEIY1uhXKplsVx3ZQrSql4JftfprRbMhRmIhsmGKgdhMS2Kg2mpWsmvl\nti9rU/39738PpBX2D3/4Q61tp512qruHmJd9AHsx2H4kclXJZLtIal7F7BV+ZOWi32UbtZmANH9i\nDueffz5Qb9PMw2PsPOt8PVfSJi644IJan49+9KMAnHXWWQAcdNBBtbbrrrsOgLe85S1Akq+1u/Zq\nsH0j2S6vZHKe9Ujyse+UtA3Jwy6yjz/+OJDeJaGkxQgl8pVnCbObY3J7p83OpW+H3mGNo5SPtFmM\n9u1+PIQwDaD6c9ly+jt6Ay7X/oXLdpwwWiZ6KfBR4PTqz0sad69HaUtnvvrYPvpdbEBeckjeNq1M\nus4DDzxQ62PZT47hMunbyp5avcSULAMRA5ZtRePo0RpLLcnV/s9iH/Km6qcNtpdmcP311wPlrbpi\nEXvuuSdQzxLEeEp29OGiQKwWIzueamZZD/Mvf/lLIDGWN7/5zUC9htKNURcN0JJshZz1i/Fvuumm\ntT7ynKuPtUnmERGNKq3qfPttyD33et8s282ZqNUK9Z7q2cujBEaD5TLREMKFwA3AFiGExSGEo6gI\nYk4I4V5gn+rfjh6Cy7V/4bJtL5rxzh82TNPeYzwWRxvhcu1fuGzbi45v/hUdzx1MpWB1havYfbI6\nT9RfKrdVx3PVzjqt8vNL6n0zKrpUCI2/FDzc77AmGDkc8gJxyt4D8Jd/+ZcAXHPNNUPO32WXXQC4\n+uqrgRTqZM0BjfauSz3T/TUeW3L5nnvuAVK+hMsuu6zWdvHFFwNJjdf47bPToyabMUFeHkRmNRha\nvrxkmssD6W2fZgrM5W02H4bMCZKrfZf1u9rkkGolXK333MYOh8PRRegIE7XMLDcUC3YL3qOPPgqk\n1cP2ldFY2zflUCoZihtls87LKltotVIojl21FITdqGDWoDBRO3d5UHReghhgwYIFQMq0ZFmm2Km2\nhL7zne8E6uc+r4pgn4s8AF/3t2FUcngodM46jVQoT+FParOOqUGRq0XOAEtz34hJ5u9eqZhcHuZY\nyh+cw8pCrFROX7txRs+BnJL6fpSy3zcLZ6IOh8PRAjpuE9UKkCcrsHYMZS5XLki77VMrobZvKnTB\notHKqPO1QomVaEsgJAas+9os2ldccQUAW265ZfGeg458O60NfNZGBQW2W5ahECeFp2nu7RZCocRm\nFPKSby+86qqran3UdvfddwPJ5m6vqUQmeYndQUXOEktzX0o8IuT91afkQ2hkc86/G5Y96vc87ND2\nz59HG4Y1UjgTdTgcjhbgH1GHw+FoAR13LOU7FwSrCsgBUQpp0U4JOSBK6kW+z9YiL5YmJ5JVz2V8\n1nXkTALYbrvtgOQUyXdODRKsqp1n15E8bejZ+9//fgC+/vWvA/WOJRUpk3xKoWOC5GrVPz1j6i/z\njHKQQlLf58+fXzdmgF/84hcArL/++nX3t46t0vPU78gL1Qk2xEjzqHexlHVNjr88VMqeV9rNlKvv\n6mtNQeqj57FkatBzIadTJ7I4ORwOh4MOMdGSoVdMRX/bve/KeC6nkc2OLpYqJlm6h5ikwp4sy9Ux\nBegq24/tI6eTxnTLLbfU2tRf0KprwyoGhZVaxiEozERzaEOElJFp8eLFdX0glTZeuHAhUA6GzlmI\nlblYiIqV7bdfJUexlYucTDr/tNNOq7WpHLM2dugZss/ZoDBRywTF2PJnuiR7vUN2ziQjzWuePa10\nj+fRZDwAAAj0SURBVFL4kcZUCk2URqOfNv+vGLB+Wu2n9P82A2eiDofD0QI6wkQbJWfWKrTbbrvV\njmnVUEadUhhTXiPJZm7JVx3LeHQt1WPReZZlyC6mY9b+k5eOzUvt5tfqZ5TYRB5SYm2iN9xwA5AC\n2S2rl2agbZpiLja8TPNakotsqvq57777AnDllVcOOX+fffYBYPvtt6+1Kfxq6tSpdeO3smwlLKaX\nULIX6vkWM7TMVGFh8i/YmmRipdJI8ryvUP9+DgfJobQlW34SXce+i2KZ2uCxbNnQjIAjtY86E3U4\nHI4W0PFg+zzYVrYsa2PRSqI2VWKElGn8tttuAxKztGxXK6FWH/2019bKVgrQFcR0SsxSTEX2tZKn\nuN/RKImEmLplGbKJfutb3wISI4SUO1be+RKr0BZOsZsbb7yx1rbjjjsCsNdeewEpd6iVqwL6P/KR\njwD1nnuNV3LVedYeX7ID9jvyuknSNKxcpfHJ/mzfRb27Ol9s3vbR7yVfgo7lERn22cu3dlqZyT6q\nPo2S2DSLZvKJzgghXBNCuDuEMD+EcFz1+JQQwpUhhHurP4fPfOzoOrhc+xMu1/ajGXX+NeCEGONs\nYBfgUyGE2XgJ1l6Hy7U/4XJtM5pJyrwUWFr9/fkQwgJgOi2UYC0VpZJqJHVBoSmQcjqqj6X5cjbJ\nmCwn0qJFi2p9ckOzzUeaO4bkuLAOEO3Lt+E5Qq6+N1JFugnjIVerNmleJU/Nh+0jh9L73vc+AH79\n61/X2qSOz549G4BvfOMbQH2RQKlicvx98IMfrLXp+dE1tUHCloqR7FQq2TqK8oxfMh10u5NwPORa\nMkflZT5s1jS9p3lOWXuezDIy15T2vpdU9XzPfu5Qtufr/jasTe+yzAp248xoMSLHUrWW9bbAjYyi\nBKuXTO5OtCpXL5ncnWhVrl4yuTk0bVUNIUwGfgIcH2N8LnOcNFWCtVQyOQ+yV5tCSyAFXIuRWiYp\nw78tmQppu6Htr9WrFAgvVqN7KZM6JKajsdr/fbgHrRTu040YC7mut956MYRQV8BP18kL1lnZywko\ndqLS1JCKESrD0lFHHVV3jr2WHIeSHSS52vAaqGebP/7xj4FUFll5ayFl7tKzU9Iwurlk8ljIVSWT\nS07SfOOK7aM5Vnlkm2NX8tNzkWtwFvn2TdtP77I0RstW9X6rioIqGEDSRFScUP+P1TDGJZ9oCGEl\nKgL5QYzx4uphL8Ha43C59idcru3FcploqCwV/wksiDGeYZrGpARrvqKUwka0ssnOaVc2rSjKC6hV\nxNo0xVC0elr7jewmCstQcLcdh1apPMDYtuVbEEtb5boJ4y1XzZVYgebchqdJZrI37r13qqP2/e9/\nH4ATTjih7jyVV4ahWdWtXHXsyCOPBODYY48F6sPbFGIlpmK1F41NLKi0ZbgbQ9fGW6569nObqH3G\nNfelulQKbs/fe/tO52GPNgetNFRtFdZ59n3Vuyx7pzRJO87cv2Lf6ZG+r82o828DPgzcGUK4vXrs\nZCrC+FG1HOvDwMEjurOj03C59idcrm1GM97564DhPs1egrVH4XLtT7hc24+O71jKDctSm6wxWaqh\nqLs1AmtnS57NxYZIyZhdUrV1LfUpmRPyfbr2fI0pz4daKsbXz8j/x7zsiubJhonlu1esGqWwJ5UO\n0V56lVKGZMqRk8LufZ83bx4Al19+OZCyQtn76xnRLhab0UeqqMYkJ5ItdDeI0SZ5iFEjlMp8WDMb\npHm1YVDqX8oYpdLMko/MCtbhKBNSSWZCvue+lLO0WfjeeYfD4WgBHWeiQr5qWbbZKG+kzdwDiR00\nWllKIVZ5ELW9Rz4muzLmoS8jXcX6HbljxoanqU1MspRnVhmB5FCwjEWB05LnzTffXGuTA0n5TOUw\ntM4nBfvL2WFZqhiOdYRBdzoJO4GcLdrnPtf4SiFSOtao8GAJuSO3lL0+3yBhNYZmSi6PFM5EHQ6H\nowV0DRMV8ozVMNRuWmKSub2ytOLkZVbtecONw6JREO4g2sdGAtmnSmFIOauBoWFl2vxg+9hr2T6Q\n2KWOKejePheSvX6WNk7kFRMc9SixzdxuWnpf8/ezlL0+v4e9T/4taJQ1rVHbWPgrnIk6HA5HC+g6\nJlpCo5UlZ6Alb1t+nl3pBsFz3k1oJBfL5vOqqWKSlrHksraRFdre5zbM9qPROzicNtdIy2skw26Q\nrzNRh8PhaAH+EXU4HI4W0BPqfDPoBlrvGHs047CT7Mei1IPDMVI4E3U4HI4WENrpWAkhPAG8CDzZ\ntpuOHdam9XFvFGNcZywG001wubpcuxBtk2tbP6IAIYRbYow7tPWmY4BeHXe70Kvz06vjbhd6dX7a\nOW5X5x0Oh6MF+EfU4XA4WkAnPqJnd+CeY4FeHXe70Kvz06vjbhd6dX7aNu6220QdDoejn+DqvMPh\ncLQA/4g6HA5HC2jrRzSEsF8IYWEI4b4QwkntvHezCCHMCCFcE0K4O4QwP4RwXPX4lBDClSGEe6s/\n1+z0WLsFLtf+hMu1yTG0yyYaQpgALALmAIuBm4HDYox3t2UATaJak3tajPG2EMLqwK3AQcARwNMx\nxtOrD9SaMcYTOzjUroDLtT/hcm0e7WSiOwH3xRgfiDG+CvwQOLCN928KMcalMcbbqr8/DywAplMZ\n63nVbudREZTD5dqvcLk2iXZ+RKcDj5i/F1ePdS1CCDOBbYEbgakxxqXVpseAqR0aVrfB5dqfcLk2\nCXcsDYMQwmTgJ8DxMcbnbFus2EA8NqwH4XLtT3RSru38iD4KzDB/b1A91nUIIaxERSA/iDFeXD38\neNX+IjvMsk6Nr8vgcu1PuFybRDs/ojcDs0IIG4cQVgYOBS5t4/2bQqgkp/xPYEGM8QzTdCnw0erv\nHwUuaffYuhQu1/6Ey7XZMbQ5Fd7+wP8DJgDnxBi/3LabN4kQwu7Ab4A7ARX0OZmKneVHwIbAw8DB\nMcanOzLILoPLtT/hcm1yDL7t0+FwOEYPdyw5HA5HC/CPqMPhcLQA/4g6HA5HC/CPqMPhcLQA/4g6\nHA5HC/CPqMPhcLQA/4g6HA5HC/j/W4KrxDincoUAAAAASUVORK5CYII=\n",
"text/plain": [
"<matplotlib.figure.Figure at 0x135f21588>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"for X_batch, y_batch in datagen.flow(X_bin_train, y_bin_train, batch_size=9):\n",
" # create a grid of 3x3 images\n",
" for i in range(0, 9):\n",
" plt.subplot(330 + 1 + i)\n",
" plt.title(\"{0}\".format(y_batch[i]))\n",
" plt.imshow(X_batch[i].reshape(28, 28), cmap=plt.get_cmap('gray'))\n",
" # show the plot\n",
" plt.show()\n",
" break"
]
},
{
"cell_type": "code",
"execution_count": 127,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# nb_classes = 2\n",
"# y_bin_train = np_utils.to_categorical(y_bin_train, nb_classes) \n",
"# y_bin_test = np_utils.to_categorical(y_bin_test, nb_classes)"
]
},
{
"cell_type": "code",
"execution_count": 128,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"input_dim = 784 #28*28 \n",
"# output_dim = nb_classes \n",
"model = Sequential() \n",
"model.add(Flatten(input_shape=X_bin_train.shape[1:]))\n",
"model.add(Dense(1, input_dim=784, activation='sigmoid')) \n",
"batch_size = 64 \n",
"nb_epoch = 20"
]
},
{
"cell_type": "code",
"execution_count": 129,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"name = datetime.datetime.now().isoformat() + \"whitened_binary\"\n",
"c = {\"lr\": 0.1, \"bs\": batch_size, \"momentum\": 0.9}\n",
"name += \"lr=\" + str(c['lr'])\n",
"name += \"m=\" + str(c['momentum'])\n",
"name += \"bs=\" + str(c['bs'])"
]
},
{
"cell_type": "code",
"execution_count": 130,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"callbacks = []\n",
"# callbacks.append(LambdaCallback(on_epoch_end=compute_hessian_stats_2))\n",
"callbacks.append(TensorBoard(log_dir=\"new/\" + name))"
]
},
{
"cell_type": "code",
"execution_count": 131,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"model.compile(optimizer=SGD(lr=c['lr'], momentum=c['momentum']), loss='binary_crossentropy', metrics=['accuracy']) "
]
},
{
"cell_type": "code",
"execution_count": 132,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(2115, 28, 28, 1)\n",
"(2115, 28, 28, 1)\n"
]
}
],
"source": [
"# whiten test according to the training set whitening transform\n",
"print(X_bin_test.shape)\n",
"for i in range(len(X_bin_test)):\n",
" X_bin_test[i] = datagen.standardize(X_bin_test[i])\n",
"print(X_bin_test.shape)"
]
},
{
"cell_type": "code",
"execution_count": 133,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'2018-01-03T18:00:45.316153whitened_binarylr=0.1m=0.9bs=64'"
]
},
"execution_count": 133,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"name"
]
},
{
"cell_type": "code",
"execution_count": 134,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/20\n",
"198/197 [==============================] - 5s 23ms/step - loss: 0.1591 - acc: 0.9636 - val_loss: 3.2950 - val_acc: 0.7934\n",
"Epoch 2/20\n",
"198/197 [==============================] - 4s 22ms/step - loss: 0.0672 - acc: 0.9921 - val_loss: 3.0727 - val_acc: 0.8061\n",
"Epoch 3/20\n",
"198/197 [==============================] - 4s 21ms/step - loss: 0.0488 - acc: 0.9931 - val_loss: 3.3909 - val_acc: 0.7882\n",
"Epoch 4/20\n",
"198/197 [==============================] - 5s 23ms/step - loss: 0.0393 - acc: 0.9957 - val_loss: 3.2591 - val_acc: 0.7948\n",
"Epoch 5/20\n",
"198/197 [==============================] - 5s 23ms/step - loss: 0.0337 - acc: 0.9976 - val_loss: 3.2963 - val_acc: 0.7939\n",
"Epoch 6/20\n",
"198/197 [==============================] - 5s 24ms/step - loss: 0.0311 - acc: 0.9979 - val_loss: 3.2629 - val_acc: 0.7962\n",
"Epoch 7/20\n",
"198/197 [==============================] - 5s 23ms/step - loss: 0.0285 - acc: 0.9982 - val_loss: 3.3642 - val_acc: 0.7887\n",
"Epoch 8/20\n",
"198/197 [==============================] - 4s 22ms/step - loss: 0.0269 - acc: 0.9981 - val_loss: 3.4124 - val_acc: 0.7863\n",
"Epoch 9/20\n",
"198/197 [==============================] - 4s 22ms/step - loss: 0.0269 - acc: 0.9983 - val_loss: 3.4178 - val_acc: 0.7863\n",
"Epoch 10/20\n",
"198/197 [==============================] - 4s 22ms/step - loss: 0.0261 - acc: 0.9983 - val_loss: 3.4138 - val_acc: 0.7858\n",
"Epoch 11/20\n",
"198/197 [==============================] - 3s 16ms/step - loss: 0.0259 - acc: 0.9984 - val_loss: 3.4029 - val_acc: 0.7872\n",
"Epoch 12/20\n",
"198/197 [==============================] - 3s 15ms/step - loss: 0.0258 - acc: 0.9984 - val_loss: 3.3989 - val_acc: 0.7877\n",
"Epoch 13/20\n",
"198/197 [==============================] - 3s 17ms/step - loss: 0.0258 - acc: 0.9984 - val_loss: 3.4019 - val_acc: 0.7872\n",
"Epoch 14/20\n",
"198/197 [==============================] - 3s 16ms/step - loss: 0.0257 - acc: 0.9984 - val_loss: 3.4006 - val_acc: 0.7872\n",
"Epoch 15/20\n",
"198/197 [==============================] - 3s 14ms/step - loss: 0.0257 - acc: 0.9984 - val_loss: 3.3970 - val_acc: 0.7877\n",
"Epoch 16/20\n",
"198/197 [==============================] - 4s 21ms/step - loss: 0.0257 - acc: 0.9984 - val_loss: 3.3977 - val_acc: 0.7877\n",
"Epoch 17/20\n",
"198/197 [==============================] - 3s 17ms/step - loss: 0.0256 - acc: 0.9984 - val_loss: 3.3963 - val_acc: 0.7877\n",
"Epoch 18/20\n",
"198/197 [==============================] - 3s 17ms/step - loss: 0.0256 - acc: 0.9984 - val_loss: 3.3960 - val_acc: 0.7872\n",
"Epoch 19/20\n",
"198/197 [==============================] - 4s 19ms/step - loss: 0.0256 - acc: 0.9984 - val_loss: 3.3946 - val_acc: 0.7877\n",
"Epoch 20/20\n",
"198/197 [==============================] - 2s 12ms/step - loss: 0.0256 - acc: 0.9984 - val_loss: 3.3916 - val_acc: 0.7872\n"
]
},
{
"data": {
"text/plain": [
"<keras.callbacks.History at 0x12472ab38>"
]
},
"execution_count": 134,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.fit_generator(datagen.flow(X_bin_train, y_bin_train, batch_size=batch_size),\n",
" steps_per_epoch=len(X_bin_train) / batch_size, \n",
" epochs=nb_epoch, \n",
" validation_data=(X_bin_test, y_bin_test), \n",
" callbacks=callbacks\n",
" )"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Non-whitened"
]
},
{
"cell_type": "code",
"execution_count": 135,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# the data, shuffled and split between train and test sets \n",
"(X_train, y_train), (X_test, y_test) = mnist.load_data()"
]
},
{
"cell_type": "code",
"execution_count": 136,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(60000, 28, 28)"
]
},
"execution_count": 136,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"X_train.shape"
]
},
{
"cell_type": "code",
"execution_count": 137,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"zeros_train = y_train == 0 \n",
"ones_train = y_train == 1\n",
"mask_train = [any(tup) for tup in zip(zeros_train, ones_train)]\n",
"\n",
"X_bin_train = X_train[mask_train]\n",
"y_bin_train = y_train[mask_train]\n",
"\n",
"zeros_test = y_test == 0 \n",
"ones_test = y_test == 1\n",
"mask_test = [any(tup) for tup in zip(zeros_test, ones_test)]\n",
"\n",
"X_bin_test = X_test[mask_test]\n",
"y_bin_test = y_test[mask_test]"
]
},
{
"cell_type": "code",
"execution_count": 138,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.53233320173707066"
]
},
"execution_count": 138,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sum(ones_train)/(sum(zeros_train)+sum(ones_train))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 139,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"X_bin_train = X_bin_train.reshape(X_bin_train.shape[0], 28, 28, 1)\n",
"X_bin_test = X_bin_test.reshape(X_bin_test.shape[0], 28, 28, 1)"
]
},
{
"cell_type": "code",
"execution_count": 140,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"input_dim = 784 #28*28 \n",
"X_train = X_train.astype('float32') \n",
"X_test = X_test.astype('float32') \n",
"X_train /= 255 \n",
"X_test /= 255"
]
},
{
"cell_type": "code",
"execution_count": 141,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# define data preparation\n",
"datagen = ImageDataGenerator()"
]
},
{
"cell_type": "code",
"execution_count": 142,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"datagen.fit(X_bin_train)"
]
},
{
"cell_type": "code",
"execution_count": 143,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAVEAAAEICAYAAADm98d9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm0FNW1x/HvFnDEAUSJAk8cgCUSH8YhRk1EESVqogaD\nolFMjEaNRkwcAPHFeYxmUKMSQwAFwUSNqFE0RMARRYwxgAJhOaCAqBjACZHz/ug+1dV36Fvd1V3d\nXff3WYvV1aequw53Q919qs5gzjlERKQ0G1S7AiIi9UwXURGRGHQRFRGJQRdREZEYdBEVEYlBF1ER\nkRh0ERURiSH1F1EzO9vMZpvZ52Y2ttr1kfIxs45m9oCZfWxmb5rZCdWuk8RXb3FtW+0KJOBd4Erg\nMGCTKtdFyutWYC3QGegLPGJmrzjn5la3WhJTXcXVWsuIJTO7EujqnDul2nWR+MxsM2Al0Mc5tyBb\nNh541zk3vKqVk5LVY1xT35yX1OoJrPP/0bJeAXarUn2kPOourrqISr1qD6xqULYK2LwKdZHyqbu4\n6iIq9WoNsEWDsi2B1VWoi5RP3cVVF1GpVwuAtmbWI1T2v0BNPnyQyOourqm/iJpZWzPbGGgDtDGz\njc2sNfRKSDXn3MfA/cDlZraZmR0AfBe4q7o1kzjqMa6pv4gCo4BPgeHAD7Lbo6paIymXs8h0W3sP\nmAicWavdYKQodRXXVtPFSUSkElpDJioiUjG6iIqIxBDrImpmA83sdTNbZGY1OZpAiqe4ppdiW34l\n3xM1szZkuiMMAJYALwJDnHPzylc9SZriml6KbWXE6eqzD7DIObcYwMwmAUcBzQbEzFr7U6z3nXPb\nVLsSLVBci1cPcYUiY6u4RotrnOZ8F+Dt0Psl2TJp3pvVrkAEimvx6iGuoNgWK1JcK97p3MxOB06v\n9HkkWYprOimuxYtzEX0H6BZ63zVblsc5NxoYDWoe1AnFNb1ajK3iWrw4zfkXgR5mtqOZbQgcD0wp\nT7WkihTX9FJsK6DkTNQ5t87MzgamkhmXPqaWh2ZJNIpreim2lZHosE81D3jJObdXtStRboqr4ppS\nkeKqEUsiIjHoIioiEoMuoiIiMegiKiISg2Z4z+ratSsAd999NwDf+ta3gn1mBsCRRx4JwCOPPJJw\n7aRYxx13HAA9e/Zs8dipU6cC8MILL1S0TpJOykRFRGLQRVREJIZW35zv1KkTAHfccQcA3/zmNwEI\n95/12+vXr0+4dhLFRRddBMBll10WlLVtm/mn7W/FFHLJJZcA8PDDDwdl3/ve98pZRSnBbrvtBuTH\nYsiQIXnHLFy4EIBnnnkmKLv33nsBeOONNypcwwxloiIiMbT6EUt9+vQB4JVXXmn2mIkTJwJwzjnn\nAPDRRx+VejqNbImpe/fuwfZDDz0EQK9evQBo06ZN2c6zZs0aALbccssohyuuRerfvz8A7du3b/aY\nBx54AMhvFTYn3OL4/PPPARg9ejQAF1xwQbBv7dq1xVRTI5ZERCqtVd4T7du3b7D9+OOPN3nMqlWr\ngu3p06cDsTJQicl3WfrlL38ZlPkMtBijRo0Ktn23Nm+DDXI5xemna0rNctlhhx2Cbd+q23vvvYHC\nrYfXXnsNgEsvvTQomzZtWt7n999/fwAuvvji4JgNN9wQgLPPPhuARx99NNj32GOPlfaXKECZqIhI\nDLqIiojE0GJz3szGAEcC7znn+mTLOgKTge7AG8Bg59zKylWzvIYNGxZsb7311k0ec+GFFwbbf/zj\nHytep6TVS1xnzpwJwNe//nUg13WpVOGHDOPGjQPgvPPOA/IfTlx++eWxzlNNtRLbvfbKPJOZMGFC\nUNajRw8g9yB39uzZAPz3v/8Njrn//vsBePbZZ5v9bt8s32677Vqsx9VXXx1sv/zyywAsX7685b9A\nRFEy0bHAwAZlw4FpzrkewLTse6kvY1Fc02osim1iInVxMrPuwMOh32qvA/2cc0vNbDtgunOuxbv8\n1eri5G9eDx48GICbb7452NehQ4e8Y88///xGx6xbt65cVamprjC1Ftdddtkl2PYPds4991ygcAa6\nYsUKIP8h4VZbbQXAEUcc0eznPvjgAwC23XbbEmscqKm4QnliW2pcfetu0aJFQH43Md8p3mf/PhMt\nlZ8b4c477wzKDjjggGaP9/sKZbkhkeJaatuos3NuaXZ7GdC5uQO1emBdUVzTK1JsFdfixe7i5Jxz\nhX5j1cLqgdtvvz2Qm6GpkLffzizLXcbssy5VI65nnnlmsB2+b90cf6/6yiuvBOCtt94K9m200UZA\n7r7c8OGZ1uvhhx8eHOOz1euuuw7IDR9Nu0KxLTWum266abD917/+FYAtttgCyJ8dy8+EFr4HGseC\nBQsAuOaaa4Ky8PDdhr761a8CkTPRSEp9Or882yQg+/pe2Wok1aS4ppdiWyGlZqJTgKHAtdnXB8tW\noyrxT4HnzJlT5ZpUVaJx3XnnnYFcC+FrX/tai595773c//2bbroJyM9APT/0z9+DGzRoEAD33HNP\ncMzRRx8NwM9//nMA5s+fH+wbO3ZstL9E/ahobL/73e8G2/vttx8Aq1evBuCUU04J9pUrA21o1qxZ\nwbbPMn09wvxkJn7CoXJoMRM1s3uA54BeZrbEzE4lE4gBZrYQOCT7XuqI4ppeim2yWsxEnXNDmtnV\nv8x1kQQpruml2CYrtWPnw12Xwp19G/I3vf3Y7HBzUcpvp512CrZ9d7J99tmnxc/5uBx//PFBmZ8B\nyMfwZz/7WbDv+eefz/u8n70nPOfonnvuCUC3bt0A2GabbSL+LcTzs2o1NdfAgw9m7hj4MfCV9OGH\nHwbb/t9IU7d5DjnkkLKfW8M+RURiSG0mGh7q5Wd6acqxxx4LKANNiv95Q+Ps5d133w22O3bsCMDG\nG28M5DLLGTNmBMf4Ttx+BvMoc3/+61//CrZvvfVWAK69NnN7cMcdd4z2l5BAly5dADjwwAMb7Qt3\nO0qS/788YsSIROqhTFREJIbUZaL+/lq/fv2qWxFp0g9/+MNGZf6epl85AHLr5Pi5KP/0pz81+pzv\nLnPaaacBxc9s77/TZ6LhzNh3k4kyQKM182uSNbWWlR+OmzTfejnrrLOA/LqVc+IRT5moiEgMuoiK\niMSQmua8b8p94xvfAHKzu4T5JT/8DDIAy5YtS6B24ud/9PNJhvnx1H5WJYB33nkHyDXnFy9e3Ox3\nl2sUTLjZF17SQprnZ94KzwbnlzH+9NNPq1InP1+C77oWngfjpJNOKvv5lImKiMSQmkzUz8jjx1M3\nxS/BmsJx0TXvsMMOA2D9+vVB2R/+8AcgN8Y6zM+s5B8w+RmewjPTf/LJJ5WprMQyb948INn4hFeo\naLgqgV/cDuDvf/972c+tTFREJIbUZKL/93//1+w+f6/t97//fVLVkQaaWkHBD/v0QzLD/JyQfrlq\nv5aOnycUSst02rVrF2yfccYZefvC987uu+++or+7NTrhhBMalVVqpqam+BaoXx4ZCs8iVQnKREVE\nYqj7TPSKK64A8mdFb+jVV18F4q/nIuV16qmnAnDLLbcA8OWXXzY6JjwTfRw+Aw2v4hqejARg8uTJ\nwXYSk2akge/YHm5p/Pvf/674ef1EIn7obrjXh+8V4NdUq0QH+7Ao84l2M7MnzWyemc01s3Oz5R3N\n7AkzW5h97dDSd0ntUFzTSXFNXpTm/DrgF8653sC+wE/NrDdagrXeKa7ppLgmLMqkzEuBpdnt1WY2\nH+gCHAX0yx42DpgOJL7Sl18gq+G46XBXhhNPPDHROtWDpON61113AfCDH/wgKPPd0XxT0B8DsHLl\nyljn811eTj75ZAD23XdfIH8WKc+f/7nnnot1zlqQdFybGjPvlyW+8cYb4349AF27dg22/S0gv6ig\nv50Q7ibnFzAML6FdSUXdE82uZb0HMAstwZoaims6Ka7JsKa6njR5oFl7YAZwlXPufjP7yDm3VWj/\nSudcwfss5VpaNzwT+sSJE4HcXJBPPfUUkMtAoOkZrqvkJefcXtWuRFjScfWd5yG3eJwXfpgTnjc0\nvC+8/K6fy7J//8arXgwcOBAoPHzT/9sfM2YM0PTs7BG12rj61kO4q9OaNWsAuOSSS4Dczze8r6G2\nbXP5nO+21Llz50af33333QH44osvgNxcoeHVK8r4/z1SXCN1cTKzdsB9wATn3P3ZYi3BWucU13RS\nXJPVYiZqmZse44APnXPDQuU3AB845641s+FAR+fchc19T/YzsTJRf98zPIGIHx7ofzNdf/31QOHO\n91VUMxlLteLq1zWC3HC8zTffvLjKx/DZZ58F2z57iZGBeq02rn4FAr+eEjReqjicGb7yyitA48EX\nfr0sgIMPPtifH8gftumHlN58880ALFq0qKUqxhEprlHuie4PnAS8amb/zJaNJLPk6r3Z5VjfBAaX\nWlOpCsU1nRTXhEV5Ov800PgRXIaWYK1Tims6Ka7Ji/xgqSwni9mc992Zmpr1x4+P33bbbeOcotJq\nptlXTqXG1TftffMtPENTc038DTbI3cYPP4yA3C0dyDUX/RIVvvn36KOPBseUcWRNq4/rJptsEmz3\n6tULgAEDBpSlHuF5DJYsWQI0Pd9CBZTvwZKIiDQtNZmoXwBt/PjxcU5Raa0+Y4lr5513DraPP/74\nvH2TJk0Ktv/zn/8kVSVQXNNKmaiISKXVVSaaAspY0klxTSdloiIilaaLqIhIDLqIiojEoIuoiEgM\nuoiKiMSgi6iISAxJL1T3PvBx9jVJO5IZT/wGsCmwC/Aa8FmBzzTUifj1bn5yy/qmuKZTteLq5z3d\ngkyi90YJ35FYXBPtJwpgZrOT7FNnZpsBK4E+zrkF2bLxwLvOucjrzCRd73qjuKZTNX8+ZnYl0NU5\nd0oJn02s3q2hOd8TWOf/o2W9AuxWpfpIeSiuUhNaw0W0PbCqQdkqILmZgKUSFFepCdW4iI5O+Hxr\nyNxbCdsSaDyLSWFJ17veKK7pVK8/n8TqnfhF1DmXdFAWAG3NrEeo7H+BucV8SRXqXVcU13Sq159P\nkvVOfXPeOfcxcD9wuZltZmYHAN8F7ir8Sallimt6mVlbM9sYaAO0MbONzSzpnkSRpf4imnUWsAmZ\nFQ4nAmc654rKWKQmKa7pNAr4FBgO/CC7PaqqNSog6anwBgK/JfMb5k7n3LWJnTwiM+sGjAc6Aw4Y\n7Zz7rZl1BCYD3cn0WxvsnFtZrXrWEsU1nRTXiHVI6iJqZm3I3McaACwBXgSGOOfmJVKBiLJrcm/n\nnJtjZpsDLwFHA6eQWYbWLznbwTl3URWrWhMU13RSXKNLsjm/D7DIObfYObcWmAQcleD5I3HOLXXO\nzclurwbmA13I1HVc9rBxZAIlimtaKa4RxbqImtlAM3vdzBZlr/aFdAHeDr1fki2rWWbWHdgDmAV0\nds4tze5aRqb5kEqKa3oVEVvFNaKSL6LZdP9W4NtAb2CImfUuV8WqzczaA/cBw5xzeZ26XeYeSCqX\nTlBc0xlXSHdsqxpX51xJf4BvAFND70cAI1o43rXyPytK/Xkn9UdxTWdci42t4ho9rnH6XjWV7n+9\n4UFmdjpweozzpMmb1a5ABIpr8eohrhAhtoprnkhxrXgHVpcZOTAatHpgmiiu6aS4Fi/Og6V3gG6h\n912zZVLfFNf0UmwrIM5F9EWgh5ntaGYbAscDU8pTLakixTW9FNsKKLk575xbZ2ZnA1PJjGgY4zTk\nru4pruml2FZG0sM+W/s9lpdcCmdRV1wV15SKFNfWMgGJiEhF6CIqIhJDzc7RVwnbbrstAMuXLw/K\nvvzyyyaPve2224Lt4cMzo+M+/vjjCtZOojj00EODbR+Xgw46CID169cH+7p1yzyEfvfddxOsnbRG\nykRFRGJoFZloz549AXjqqaeA/OwznL2E/eQnPwm2P/nkEwCuvTYzneLKlZpuMmnHHHMMABMnTgzK\n2rVrB+Ri2NRD0hNPPBGANWvWAPDggw9WtJ5Smr59+wbbjz/+OABbb701ABtskMv1+vXrB8CMGTOS\nq1wLlImKiMSgi6iISAypbc7vueeewfY999wDQIcOHUr6rvPOOw+AdevWAXDTTTcF+z744INSqygF\ndOrUCYChQ4cCcOmllwK5JnxLXnzxRSDXJPS3cPzDKICbb765LHWV+I46Kjffs/9/6m/TPP3008G+\n1157LdmKRaBMVEQkhtRlogcccAAAt9xyS1DWvXv3snz3BRdcAICZBWU33HADoIdN5XbkkUcCcN11\n1+WV+9YAwM9+9jMgF48BAwYE+w477DAA2rZtm/d69dVXB8f4z/3ud78ra92lvPz/aYBevXoB+d0U\nq02ZqIhIDKnJRH3XB5+59O4dbdWDFStWADBz5kwABg0a1OJnzj///GDbd7lRJhrfkCFDgu3wfWeA\nuXMz82TcfvvtQdno0aPzjrnjjjuC7bfeegvIdbbfeeedAdhkk02CY0477TRAmajEo0xURCQGXURF\nRGJosTlvZmOAI4H3nHN9smUdgclAd+ANYLBzrirtWd+Mv+aaawDYY489WvyMf0AEuea8v1EdpTmf\nBrUUV9+lZeTIkUHZFltsAcCnn34K5LomPfroo5G+098a8N8zadIkADbbbLPgGN+0992p3n///dL+\nAjWmlmLbGkTJRMcCAxuUDQemOed6ANOy76W+jEVxTauxKLaJaTETdc7NNLPuDYqPAvplt8cB04GL\nylivJvXp0wfIf7Cz6667AtEy0HPOOQfIfwDhhbvHFOPKK68E4OSTTwZg1apVhQ6vGbUUV/+AyMcS\nchnoiBEjgOgZqPfMM8/kvR81ahQAv/71r4My3/XtpJNOarSvntVSbKMKdxsMj5UHGD9+fLDtHwDX\nklKfznd2zi3Nbi8DOjd3oJZgrSuKa3pFiq3iWrzYXZycc67QMgLlWIJ1p512AmDMmDFA/owvUZx5\n5pkA3HXXXc0eM2fOHAB+9atfAfnZbiGHH344ABtttFFRdap1ScTVd6Lu379/o30+kwwPmojj5Zdf\nBuDzzz8PynzMhg0bBqQnE21JodhWa8nk8AxcDWdWS3IJo1KU+nR+uZltB5B9fa98VZIqUlzTS7Gt\nkFIz0SnAUODa7GtFJ2ls3749UFwG6rNPgAkTJgCwdu3aZo/3E4nMmzevqLoNHjwYSE1n+0Tj6u+B\n+ifoYQ2He8blJ7EI3ys9+OCDy3qOGpdobFuTFjNRM7sHeA7oZWZLzOxUMoEYYGYLgUOy76WOKK7p\npdgmK8rT+SHN7Gp8I0vqhuKaXoptslIzdr6hN954I9j+7LPPKnaeRYsWAfmzC0k0l1xySd77//zn\nP8H24sWLE6uHv52w7777BmXPP/98YueXwo499thg23dPnDVrVrWq04iGfYqIxFCzmWibNm2C7aYW\nrGpJuPNuFL67y1ZbbVX0uSS68IoDPgP0sQp3qn7zzTcrcv7wvwu/7esRrpsy0doRHqpbi10JdaUQ\nEYmhZjPRb33rW8H21KlTgeaXNw7z80iuXr26pPP5eSyjnEuKF77v6Luu+c7UPs6VFO643bATt2Je\nPYWGfYbfF9vCTIIyURGRGHQRFRGJoWab80888USwHaWZ9frrrwO5xctqqQuE1LZPPvkEgAULFlS5\nJq1XobHzzR1XK5SJiojEULOZaLH8DE9PPvlki8f6+SMh95ttt912q0zFJE+4E72fMzS8eFyl+Ada\n4Qdbnp/3YNq0aRWvh6SPMlERkRhSk4n6Lkq+m0x4NqahQ4cCubV0wjMEqVtLssIz1PsMsJKZ6Lbb\nbgvA5ZdfDsCmm27a6JjbbrutYueX9FMmKiISQ81moldccUWwffHFF7d4/BFHHAHkhovOnz8/2Ofv\ngfpMVGrTeeedF2z7+WBLXbPKx9q3QpqaO9QPzBg7dmxJ5xCBaPOJdjOzJ81snpnNNbNzs+UdzewJ\nM1uYfe1Q+epKuSiu6aS4Ji9Kc34d8AvnXG9gX+CnZtYbLcFa7xTXdFJcExZlUualwNLs9mozmw90\nocJLsN5+++3BdpTmvDdw4MC813K45557AHjsscca7VuyZEnZzpOkasU1bMWKFQBsv/32ABx33HGN\njjnjjDOAaHMhdOiQS64mT54MwIEHHph3jO9WBbl5EpYtW1ZMtWtaLcS1FPU8dr6oe6LZtaz3AGah\nJVhTQ3FNJ8U1GRZ1GJWZtQdmAFc55+43s4+cc1uF9q90zhW8z1LMEqzheQNPPvlkAG699daoHy8o\n/JutuS5Of/nLX4Lt4cMzLR//ICKGl5xze8X9knJKOq5hfsbyu+++G4C2bRv/Tp8+fTqQy/jDSx77\nOT/79esHQM+ePYN9e++9d973+M9ddFEu+SrXcsworrF179492PaLCfruaeH/rwcddBAAM2fOTKJa\nkeIaqYuTmbUD7gMmOOfuzxZrCdY6p7imk+KarBab85a5CfFHYL5z7qbQroouwRrOOHym4pUrIy0k\nPLN6GTLQmlOtuIb5bL9du3YAjBw5MtjXo0cPIJdlNuXUU08FcvfJwq2qL774AoDZs2cDcMMNNwAw\nZcqUclS9ZtVCXEsRXhPN388+55xzqlSb4kS5J7o/cBLwqpn9M1s2kkww7s0ux/omMLgyVZQKUVzT\nSXFNWJSn808DzT0S0xKsdUpxTSfFNXmRHyyV5WQxb1RvuOGGAGy55ZZB2VlnnQUU1w2qqQdLM2bM\nAOCEE04A8rvCrFmzpsQaN1JzDyDKoRIPIIYMySyd7hePGzZsWKHzA3DZZZcFZS+99BIAjzzySLmr\n1hTFtYz69u0LwEMPPQTkusBBbiTb6NGjk6hK+R4siYhI0+oqE00BZSzppLhWgJ8j2M9/ADBq1CgA\nrrnmmiSqoExURKTSlIkmSxlLOimu6aRMVESk0nQRFRGJQRdREZEYdBEVEYlBF1ERkRh0ERURiSHp\nhereBz7OviZpRzLjid8ANgV2AV4DPiviOzoRv947xPx8rVJc00lxjSDRfqIAZjY7yT51ZrYZsBLo\n45xbkC0bD7zrnIu8zkzS9a43ims6Ka4taw3N+Z7AOh+QrFeA3apUHykPxTWd6i6ureEi2h5ouHj5\nKmDzKtRFykdxTae6i2s1LqKJzGEVsgbYokHZlkDLy0fmS7re9UZxTSfFtQWJ3xNNWugey27OuYXZ\nsruAd4q5xyK1RXFNp3qMa+ovogBmNglwwI/JLCH7CLCfc25uVSsmsSiu6VRvcW0N90QBzgI2IbPC\n4UTgzFoNiBRFcU2nuoprohdRMxtoZq+b2SIzSyw1d8596Jw72jm3mXPuf5xzEwvUsZuZPWlm88xs\nrpmdmy3vaGZPmNnC7GvBNbtbE8U1nRTXaBJrzptZG2ABMABYArwIDHHOzUukAhFl1+Tezjk3x8w2\nB14CjgZOAT50zl2b/QfVwTl3URWrWhMU13RSXKNLMhPdB1jknFvsnFsLTAKOSvD8kTjnljrn5mS3\nVwPzgS5k6joue9g4MoESxTWtFNeIYl1Ei0z3uwBvh94vyZbVLDPrTubG9iygs3NuaXbXMqBzlapV\ncYprehURW8U1opIvotl0/1bg20BvYIiZ9S5XxarNzNoD9wHDnHN5nX9d5h5IKrs1KK7pjCukO7ZV\njatzrqQ/wDeAqaH3I4ARLRzvWvmfFaX+vJP6o7imM67FxlZxjR7XOLM4NZXuf73hQWZ2OnB6jPOk\nyZvVrkAEimvx6iGuECG2imueSHGt+FR4zrnRZIdgafXA9FBc00lxLV6cB0vvAN1C77tmy6S+Ka7p\npdhWQJyL6ItADzPb0cw2BI4HppSnWlJFimt6KbYVUHJz3jm3zszOBqYCbYAxroaHZkk0imt6KbaV\nkegEJLrHwksuhbOoK66Ka0pFimtrmYBERKQidBEVEYkh6dU+a9aTTz4JQL9+/QCYPn16sO+ggw6q\nQo0kbM899wTg2GOPDco6deoEwF133QXAzJkzk6+YtHrKREVEYmj1D5ai/P3NrFyn0wOIIu2yyy4A\nPPHEEwB079690THLly8H4KSTTgrK/PEJUVwr6Ljjjgu2R4wYAUCfPn2AXMvx6aefrsSp9WBJRKTS\ndBEVEYmhVT1Y8qm/f4hU7OfCD5skGf5BUlPNeK9z58xUkX/729+CskMPPRQoPtZSfTvvvDMAp556\nKgC/+MUvgn3t2rXLO3bq1KmNjpkwYQIAq1cXu8pyaZSJiojE0CoeLPlM8pe//GXe+6h8BlqGrk56\nAFGk1157DYBevXoV9bkVK1YA8KMf/QiA5557DoAPPvigjLULKK5lNGfOHAD69u1b0ucXL14MwMUX\nXwzA5MmTS62KHiyJiFRaajPRcLZZrvtil112GQCXXnppqV+hjCWiAw44AIAHHngAyHWsD3v44YcB\n+PzzzwEYNGhQs9/nu8B8//vfD8qWLVtWnsoqrmX19tuZeaO7dIm3pNOqVZlVQqZMyU1UdfLJJxfz\nFcpERUQqTRdREZEYWuziZGZjgCOB95xzfbJlHYHJQHfgDWCwc25l5aoZnW9q+4dILfFNdK/Q5/y+\nGM35mlHrcfUP8Ro242fNmhVs+xFKX375JQArV+aq+uMf/zjvc/72wJ///OegzDfty9isrwm1Htum\nDB06NNjeeuuty/KdW2yxBQBHHXVUUOb/XZWz61uUTHQsMLBB2XBgmnOuBzAt+17qy1gU17Qai2Kb\nmBYzUefcTDPr3qD4KKBfdnscMB24qIz1KloxGWi4q1LDDvT+e5J84FYNtRjXcAYycGDDa0DGxx9/\nHGx/9NFHefvOPPPMYNt3jfJlvgO3z0gh15n7qquuilPtmlOLsW3OxhtvDMDo0aODsoYd6gvxDx77\n9+8flPkM1Nt8882D7R122KGkehZS6oilzs65pdntZUDn5g7UEqx1RXFNr0ixVVyLF3vYp3POFeoK\nUeklWBt2pC/EZ6AavtmyasR1yy23DLb322+/Jo8566yzmv38unXrgu0bb7wRyGUqfnignxUKYPjw\nTIt2/PjxQK5rTdoVim3SSyb7DvHFZJ+Q67bku7X5OEP+PdAklPp0frmZbQeQfX2vfFWSKlJc00ux\nrZBSM9FgcOX2AAAHr0lEQVQpwFDg2uzrg2WrUZGiZKD+Cbwy0BZVNa6nn958K/Lee+8F4K233irq\nO/0QwGOOOQbIn6SkW7fMEuz+vunIkSOL+u46UzP/ZwG+/e1vA7n5QQu59dZbg20/0YjvkeEz2ULZ\n54cffhhsz58/v/jKtqDFTNTM7gGeA3qZ2RIzO5VMIAaY2ULgkOx7qSOKa3optsmK8nR+SDO7+jdT\nLnVAcU0vxTZZdTmfaHhcfHMzMoWb7sV0ji90bMOO+VJeBx54YLP7XnjhBQA+/fTTkr773//+N5A/\ni5NvzvsHTOFbBbfffntJ55HC/G0V/zBvgw2abwz7hQcvuOCCoGzt2rUAbLTRRgBceOGFLZ7z0Ucf\nDbbDgzXKRcM+RURiqMtMtNDDpFLn/ozSVUoPpiqjbdvMP8NCWUm5fPHFF43K/EKEZVyQUEJ23333\nYHvMmDEAbLbZZs0e74fhjho1CoDPPvss2OcHS/iHTeGO9A1NmjQJgJ/+9KelVDsyZaIiIjHUVSbq\ns8VCM9PPmDGjpO8sNCGBukhV1hlnnAHAPvvsU/FznXLKKcH23LlzK34+yQ25hfwBFc25++67AfjX\nv/4FwBFHHBHsu/POO4HculpN8XG9+uqrgdy8opWiTFREJAZdREVEYqir5nyhJrdvahc712dz31lq\nFykpr4ULFwK5EUtSP/zD3SjdxcKLyV133XVArhnvl0CG3MO/hrOshWf38p8rdnRbqZSJiojEUBeZ\naJQljovp0hRlVusyLI8sZbBmzRqgfDMshR9SSGX5bkjbbLNNi8f6pa0Bxo4dC+QvKug1zED9GPrw\nsUlloJ4yURGRGOoiE43bAb5hR/pCma0y0HTx81QOGzYMaHrorh8WGF5/SZIVng/00EMPbfF4n4H6\nFRCmTZtWmYpFoExURCSGms1Eo0wyAs13rg8/Udes97Xt+eefB/Lve/rJQbbffnsgN9zv6aefLuq7\n/Uz4119/fYvHvP/++0V9tzTNr8Laq1evyJ+Jkn0CvPPOOwD07t0bgNWrVxdZu/KLMp9oNzN70szm\nmdlcMzs3W97RzJ4ws4XZ1w6Vr66Ui+KaTopr8qI059cBv3DO9Qb2BX5qZr3REqz1TnFNJ8U1YVEm\nZV4KLM1urzaz+UAXKrwEa5RuTZBrtvtXPydl1M+31mZ8teLalNmzZwP5MfBNQj9G+rjjjgNg/fr1\nzX7PeeedF2zvvffeAHzlK1/JOyY8H6lfDmTJkiWlVr3m1EJc/axc5ZoVK9wR/6KLMlWuhWa8V9Q9\n0exa1nsAs9ASrKmhuKaT4pqMyBdRM2sP3AcMc86tCv+WqeYSrA073xZS6lyjaVZLcQ0/DPSZqHf2\n2WfnvZYq/JDxN7/5TazvqmW1FNdSzZkzB8gtfw3Jd6SPIlIXJzNrRyYgE5xz92eLtQRrnVNc00lx\nTZa1lMlZ5lfYOOBD59ywUPkNwAfOuWvNbDjQ0TlXcMGTUn+zFZNteuH7azU0H+hLzrm9ql0JqI24\nNhS+f+m7rvXs2TPWd/o1dcaNGwfA6NGjg32+w3YZKK4hvnvaM888A0DXrl1b/Ex4WWM/+MHPSF/p\n+UALiBTXKM35/YGTgFfN7J/ZspFklly9N7sc65vA4FJrKlWhuKaT4pqwKE/nnwaae8ymJVjrlOKa\nTopr8mp2xFKYfxBUzBIeNdB0lyL5BcoABg0aBMB3vvMdILfUQyHhReiGDh0KwD/+8Q8Ali9fXrZ6\nSmF+5Nltt90G5M9X4Ls/+Vt0fgnro48+Ojjm2WefTaSe5aKx8yIiMbT4YKmsJ6tyl4kaUDMPIMpJ\ncVVcC/Ed5AF23XVXAF5++WUAfvvb35bjFJUSKa7KREVEYlAmmixlLOmkuKaTMlERkUrTRVREJAZd\nREVEYtBFVEQkBl1ERURi0EVURCSGpId9vg98nH2tN52IX+8dylGRGqS4ppPiGkGi/UQBzGx2Pfap\nq9d6J6Vefz71Wu+k1OvPJ8l6qzkvIhKDLqIiIjFU4yI6uuVDalK91jsp9frzqdd6J6Vefz6J1Tvx\ne6IiImmi5ryISAy6iIqIxJDoRdTMBprZ62a2KLviYM0xs25m9qSZzTOzuWZ2bra8o5k9YWYLs68d\nql3XWqG4ppPiGrEOSd0TNbM2wAJgALAEeBEY4pybl0gFIsquyb2dc26OmW0OvAQcDZxCZhlav+Rs\nB+fcRQW+qlVQXNNJcY0uyUx0H2CRc26xc24tMAk4KsHzR+KcW+qcm5PdXg3MB7qQqeu47GHjyARK\nFNe0UlwjSvIi2gV4O/R+SbasZplZd2APYBbQ2Tm3NLtrGdC5StWqNYprOimuEenBUjPMrD1wHzDM\nObcqvM9l7oGob1gdUlzTqZpxTfIi+g7QLfS+a7as5phZOzIBmeCcuz9bvDx7/8Xfh3mvWvWrMYpr\nOimuESV5EX0R6GFmO5rZhsDxwJQEzx+JmRnwR2C+c+6m0K4pwNDs9lDgwaTrVqMU13RSXKPWIeHV\nPg8HfgO0AcY4565K7OQRmdkBwFPAq8D6bPFIMvdZ7gX+B3gTGOyc+7Aqlawxims6Ka4R66BhnyIi\npdODJRGRGHQRFRGJQRdREZEYdBEVEYlBF1ERkRh0ERURiUEXURGRGP4fbAIyWWv8KvIAAAAASUVO\nRK5CYII=\n",
"text/plain": [
"<matplotlib.figure.Figure at 0x13d01bfd0>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"for X_batch, y_batch in datagen.flow(X_bin_train, y_bin_train, batch_size=9):\n",
" # create a grid of 3x3 images\n",
" for i in range(0, 9):\n",
" plt.subplot(330 + 1 + i)\n",
" plt.title(\"{0}\".format(y_batch[i]))\n",
" plt.imshow(X_batch[i].reshape(28, 28), cmap=plt.get_cmap('gray'))\n",
" # show the plot\n",
" plt.show()\n",
" break"
]
},
{
"cell_type": "code",
"execution_count": 144,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# nb_classes = 2\n",
"# y_bin_train = np_utils.to_categorical(y_bin_train, nb_classes) \n",
"# y_bin_test = np_utils.to_categorical(y_bin_test, nb_classes)"
]
},
{
"cell_type": "code",
"execution_count": 145,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"input_dim = 784 #28*28 \n",
"# output_dim = nb_classes \n",
"model = Sequential() \n",
"model.add(Flatten(input_shape=X_bin_train.shape[1:]))\n",
"model.add(Dense(1, input_dim=784, activation='sigmoid')) \n",
"batch_size = 64 \n",
"nb_epoch = 20"
]
},
{
"cell_type": "code",
"execution_count": 146,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"name = datetime.datetime.now().isoformat() + \"binary\"\n",
"c = {\"lr\": 0.1, \"bs\": batch_size, \"momentum\": 0.9}\n",
"name += \"lr=\" + str(c['lr'])\n",
"name += \"m=\" + str(c['momentum'])\n",
"name += \"bs=\" + str(c['bs'])"
]
},
{
"cell_type": "code",
"execution_count": 147,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"callbacks = []\n",
"# callbacks.append(LambdaCallback(on_epoch_end=compute_hessian_stats_2))\n",
"callbacks.append(TensorBoard(log_dir=\"new/\" + name))"
]
},
{
"cell_type": "code",
"execution_count": 148,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"model.compile(optimizer=SGD(lr=c['lr'], momentum=c['momentum']), loss='binary_crossentropy', metrics=['accuracy']) "
]
},
{
"cell_type": "code",
"execution_count": 149,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(2115, 28, 28, 1)\n",
"(2115, 28, 28, 1)\n"
]
}
],
"source": [
"# whiten test according to the training set whitening transform\n",
"print(X_bin_test.shape)\n",
"for i in range(len(X_bin_test)):\n",
" X_bin_test[i] = datagen.standardize(X_bin_test[i])\n",
"print(X_bin_test.shape)"
]
},
{
"cell_type": "code",
"execution_count": 150,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'2018-01-03T18:02:13.299865binarylr=0.1m=0.9bs=64'"
]
},
"execution_count": 150,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"name"
]
},
{
"cell_type": "code",
"execution_count": 151,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/20\n",
"198/197 [==============================] - 1s 4ms/step - loss: 0.7691 - acc: 0.9520 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 2/20\n",
"198/197 [==============================] - 0s 2ms/step - loss: 0.1680 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 3/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1678 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 4/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1680 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 5/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1680 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 6/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1678 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 7/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1678 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915: 0s - loss: 0.1706 - acc: 0.98\n",
"Epoch 8/20\n",
"198/197 [==============================] - 0s 2ms/step - loss: 0.1678 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 9/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1680 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 10/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1680 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 11/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1678 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 12/20\n",
"198/197 [==============================] - 0s 2ms/step - loss: 0.1678 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 13/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1678 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 14/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1678 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 15/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1678 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 16/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1683 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 17/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1678 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 18/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1678 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 19/20\n",
"198/197 [==============================] - 0s 3ms/step - loss: 0.1680 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n",
"Epoch 20/20\n",
"198/197 [==============================] - 1s 3ms/step - loss: 0.1682 - acc: 0.9896 - val_loss: 0.1372 - val_acc: 0.9915\n"
]
},
{
"data": {
"text/plain": [
"<keras.callbacks.History at 0x123737f98>"
]
},
"execution_count": 151,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.fit_generator(datagen.flow(X_bin_train, y_bin_train, batch_size=batch_size),\n",
" steps_per_epoch=len(X_bin_train) / batch_size, \n",
" epochs=nb_epoch, \n",
" validation_data=(X_bin_test, y_bin_test), \n",
" callbacks=callbacks\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment