Keras Neural Network on MNIST HyperParameter Tuning example

mnist_hyperopt_talos.py

"""
Keras and talos using Mnist as toy example
"""

import talos
from keras import Sequential
from keras.activations import relu, elu, tanh, softmax
from keras.layers import Conv2D, BatchNormalization, MaxPool2D, Flatten, Dense, Dropout
from keras.losses import logcosh, binary_crossentropy
from keras.optimizers import Adam, RMSprop
from talos.metrics.keras_metrics import fmeasure_acc
from talos.model import lr_normalizer


def create_mnist_model(x_train, y_train, x_test, y_test, params: dict):
    model = Sequential()
    model.add(Conv2D(input_shape=(28, 28, 1), strides=4, filters=96, kernel_size=(3, 3), padding='valid',
                     activation=params['activation_1']))
    model.add(BatchNormalization())
    model.add(MaxPool2D(pool_size=(3, 3), strides=2))
    model.add(Conv2D(filters=params['filter_1'], kernel_size=(5, 5), padding='same', activation=params['activation_2']))
    model.add(BatchNormalization())
    model.add(MaxPool2D(pool_size=(3, 3), strides=2))
    model.add(Conv2D(filters=params['filter_2'], kernel_size=(3, 3), padding='same', activation=params['activation_3']))
    model.add(BatchNormalization())
    model.add(Conv2D(filters=params['filter_3'], kernel_size=(3, 3), padding='same', activation=params['activation_4']))
    model.add(BatchNormalization())
    model.add(Conv2D(filters=params['filter_4'], kernel_size=(3, 3), padding='same', activation=params['activation_5']))
    model.add(BatchNormalization())
    model.add(MaxPool2D(pool_size=(2, 2), strides=2))
    model.add(Flatten())
    model.add(Dense(params['dense_1'], activation=params['dense_activation_1']))
    model.add(Dropout(params['dropout_1']))
    model.add(Dense(params['dense_1'], activation=params['dense_activation_2']))
    model.add(Dropout(params['dropout_2']))
    model.add(Dense(10, activation=params['final_activation']))

    model.compile(optimizer=params['optimizer'](lr=lr_normalizer(params['lr'], params['optimizer'])),
                  loss=params['losses'],
                  metrics=['acc', fmeasure_acc]
                  )
    out = model.fit(x_train, y_train,
                    batch_size=params['batch_size'],
                    epochs=params['epochs'],
                    validation_data=[x_test, y_test],
                    verbose=0)

    return out, model


parameter_search_space = dict(
    activation_1=(relu, elu),
    filter_1=(42, 56, 70, 84, 98),
    activation_2=(relu, elu),
    filter_2=(42, 56, 70, 84, 98),
    activation_3=(relu, elu),
    filter_3=(42, 56, 70, 84, 98),
    activation_4=(relu, elu),
    filter_4=(42, 56, 70, 84, 98),
    activation_5=(relu, elu),
    dense_1=(128, 256, 512),
    dense_activation_1=(tanh, softmax),
    dropout_1=(0.3, 0.5, 0.7),
    dense_2=(128, 256, 512),
    dense_activation_2=(tanh, softmax),
    dropout_2=(0.3, 0.5, 0.7),
    final_activation=(tanh, softmax, relu),

    optimizer=(Adam, RMSprop),
    lr=(0.5, 2, 5, 10),
    losses=(logcosh, binary_crossentropy),
    epochs=[3],
    batch_size=(16, 32)

)

if __name__ == '__main__':
    x_dataset = """Insert X data set here"""
    y_dataset = """Insert Y data set here"""

    ta_scan_model = talos.Scan(
        x=x_dataset,
        y=y_dataset,
        model=create_mnist_model,
        params=parameter_search_space,
        dataset_name='mnist',
        experiment_no='1',
        grid_downsample=.01
    )

    # EXTRACTING the hyperparameter space boundary so that you can create this model without searching again.
    print(ta_scan_model.params)

    # Accessing results of the scan

    print(ta_scan_model.data.head())
    print(ta_scan_model.peak_epochs_df)
    print(ta_scan_model.details)

    # See saved models
    print(ta_scan_model.saved_models)
    print(ta_scan_model.saved_weights)

    # See Report
    report = talos.Reporting(ta_scan_model)

    # Deploying the model
    talos.Deploy(ta_scan_model, 'mnist')

    # Or if you wish to restore a previously saved model
    # ta_scan_model = talos.Restore('mnist.zip')

    # Make predictions
    x_dataset_validate = "Insert validation dataset here"
    y_dataset_validate = "Insert validation dataset here"
    ta_scan_model.model.predict(x_dataset_validate)
    evaluator = talos.Evaluate(ta_scan_model)
    evaluator.evaluate(x_dataset_validate, y_dataset_validate, folds=10, average='macro')
    
    """ Final Steps: Using the parameters we can obtain from the search, we can recreate the model but using these parameters
    Then using these parameters, train on a larger amount of epochs"""