Neural network

digit_test.py

import numpy as np

from numpy.testing import assert_almost_equal, assert_array_equal

from sklearn.base import clone
from sklearn.cross_validation import train_test_split
from sklearn.datasets import load_digits
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.preprocessing import LabelBinarizer
from NeuralNetwork import NeuralNetwork
np.seterr(all='warn')

digits = load_digits()
X = digits.data
y = digits.target
X -= X.min()
X /= X.max()

def test_fit():
    nn = NeuralNetwork([64, 100, 10],'relu', epochs=60000, learning_rate=0.05)
    X_train, X_test, y_train, y_test = train_test_split(X, y)
    labels_train = LabelBinarizer().fit_transform(y_train)
    labels_test = LabelBinarizer().fit_transform(y_test)

    nn.fit(X_train,labels_train)
    predictions = []
    for i in range(X_test.shape[0]):
        o = nn.predict(X_test[i] )
        predictions.append(np.argmax(o))
    print confusion_matrix(y_test,predictions)
    print classification_report(y_test,predictions)

test_fit()

NeuralNetwork.py

import numpy as np
from numpy.random import random as rand

class NeuralNetwork:

    activations = {
        'tanh': lambda x: np.tanh(x),
        'relu': lambda x: np.clip(x, 0, np.max(x))
    }
    derivatives = {
        'tanh': lambda x: 1.0 - np.tanh(x)**2,
        'relu': lambda x: (np.sign(x) + 1)*0.5
    }

    def __init__(self, layers, activation='tanh', learning_rate=0.2, reg=0.0001,
                 epochs=10000):
        self.activation = NeuralNetwork.activations[activation]
        self.activation_deriv = NeuralNetwork.derivatives[activation]
        self.learning_rate, self.epochs, self.reg = learning_rate, epochs, reg

        self.initialize_weights(layers)

    def initialize_weights(self, layers):
        self.weights = []
        i = 0
        for i in range(0, len(layers)-2):
            self.weights.append((2*rand((layers[i]+1, layers[i+1] + 1))-1)*0.25)
        self.weights.append((2*rand((layers[-2]+1, layers[-1]))-1)*0.25)

    def regularize(self, weights):
        return self.reg * np.sum(weights)

    def fit(self, X, y):
        X = np.atleast_2d(X)
        X = np.hstack((X, np.ones([1, X.shape[0]]).T))
        y = np.array(y)

        for k in range(self.epochs):
            i = np.random.randint(X.shape[0])
            a = [np.atleast_2d(X[i])]

            for l in range(len(self.weights)):
                    a.append(self.activation(np.dot(a[l], self.weights[l])))
            deltas = [(y[i] - a[-1]) * self.activation_deriv(a[-1])]

            for l in range(len(a) - 2, 0, -1): # begin at the second to last layer
                deltas.append(deltas[-1].dot(self.weights[l].T)*
                                             self.activation_deriv(a[l]))
            deltas.reverse()
            for i in range(len(self.weights)):
                self.weights[i] += self.learning_rate * (
                    a[i].T.dot(deltas[i]) - self.regularize(self.weights[i])
                )

    def predict(self, X):
        X = np.append(X, 0)
        for l in range(0, len(self.weights)):
            X = self.activation(np.dot(X, self.weights[l]))
        return X

if __name__ == '__main__':
    nn = NeuralNetwork([2,5,1], 'relu', epochs=100000, learning_rate=0.5, reg=0)
    X = np.array([[0, 0],
                  [0, 1],
                  [1, 0],
                  [1, 1]])
    y = np.array([0, 1, 1, 0])
    nn.fit(X, y)
    for i in [[0, 0], [0, 1], [1, 0], [1,1]]:
        print(i,nn.predict(i))