Classification of MNIST dataset by ANN using Keras

keras_ann_mnist.py

import numpy as np

from keras.models import Sequential
from keras.layers.core import Dense,Activation,Dropout
from keras.datasets import mnist
from keras.utils import np_utils

no_epochs = 30
batch_size=150 #1000
v_length = 784
num_classes=10

#loading data
(trainData,trainLabels),(testData,testLabels) = mnist.load_data()
print(trainData.shape,trainLabels.shape,testData.shape,testLabels.shape)

#reshaping Datasets
trainData = trainData.reshape(60000,784).astype('float32')/255
testData = testData.reshape(10000,784).astype('float32')/255
print(testLabels)

#convert class vectors to binary class matrices --> one-hot encoding
mTrainLabels = np_utils.to_categorical(trainLabels, num_classes)
mTestLabels = np_utils.to_categorical(testLabels, num_classes)
print(mTrainLabels.shape)

#create a model
model = Sequential()
model.add(Dense(512,activation="relu",input_shape=(784,)))
model.add(Dropout(0.2))
model.add(Dense(256,activation="relu",input_shape=(512,1)))
model.add(Dropout(0.2))
model.add(Dense(10,activation="softmax",input_shape=(256,1)))

from keras import losses
from keras import optimizers

#decay = weight decay to overcome overfitting (limits weight)
#lr = learning rate
sgd = optimizers.SGD(lr=0.01, decay=0.001, momentum=0.9, nesterov=True)
adam = optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.002, amsgrad=True)
model.compile(loss=losses.categorical_crossentropy, optimizer=adam, metrics=["accuracy"])

# fit the model
model.fit(trainData,
        mTrainLabels,
        validation_data=(testData, mTestLabels),
        batch_size=batch_size,
        nb_epoch=no_epochs,
        verbose=2)


#best after 10 epochs
"""
all activation are relu and output layer has softmax
0.9767 using 512,256 hidden layers, batch_size=150,loss=categorical_crossentropy, optimizer sgd with lr=0.01,weight_decay=1e-6
0.9786 using 512,256 hidden layers, batch_size=150,loss=categorical_crossentropy, optimizer adam with lr=0.015, weight_decay=0.001,, amsgrad=False
0.9789 using 512,256 hidden layers, batch_size=150,loss=categorical_crossentropy, optimizer adam with lr=0.015, weight_decay=0.001, amsgrad=True
0.9636 using 512,256 hidden layers, batch_size=150,loss=mean_squared_error, optimizer adam with lr=0.015, weight_decay=0.001, amsgrad=True
0.9782 using 512,256,64 hidden layers, batch_size=150,loss=categorical_crossentropy, optimizer adam with lr=0.015, weight_decay=0.001, amsgrad=True
0.9763 using 512,256,64 hidden layers, batch_size=128,loss=categorical_crossentropy, optimizer adam with lr=0.015, weight_decay=0.001, amsgrad=True
0.9762 using 512,256,64 hidden layers, batch_size=150,loss=categorical_crossentropy, optimizer sgd with lr=0.015, weight_decay=0.001
0.9809 using 512,256,64 hidden layers, batch_size=150,loss=categorical_crossentropy, optimizer adam with lr=0.01, weight_decay=0.002, amsgrad=True
0.9673 using 512,256 hidden layers, batch_size=150,loss=categorical_crossentropy, optimizer sgd with lr=0.01, weight_decay=0.002, amsgrad=True
0.9818 using 512,256 hidden layers, batch_size=150,loss=categorical_crossentropy, optimizer adam with lr=0.01, weight_decay=0.002, amsgrad=True
"""
#best after 30 epochs (overfitting)
"""
0.9831
"""
#best after 12 epochs (is ok)
"""
0.9834
"""