Learning of a Continuous Perceptron as described in the book - 'Introduction to Artificial Neural Networks' by Jacek M. Zurada

continuous_perceptron.py

"""
This is an implmentation of a Continuous Perceptron trained using Delta Learning Rule or Backpropagation.
It is based on the description of the same as given in the book - 
'Introduction to Artificial Neural Networks' by Jackek M. Zurada

Date:   July ‎16, ‎2018
Author: Pranav Gajjewar
Course: Artifical Neural Networks

The following work was done while studying as an undergraduate at SGGSIE&T, Nanded.

The work is licensed under the terms of the MIT License.
"""

import numpy as np

class ContinuousPerceptron:

    def __init__(self, x_input, y_input, learning_rate=0.1, e_max=0.01):
        '''
        x_input = data points
        y_input = target labels
        '''
        if len(x_input) != len(y_input):
            raise ValueError('X and Y must have length.')

        self.n_dims = x_input.shape[-1]
        self._lambda = 1
        self.eta = 1
        self.aug_x = self.augment_input(x_input)
        self.e_max = e_max
        self.y_input = y_input
        self.cycle_wise_errors = []
        self.__activation = self.__sigmoid

        self.__dev_activation = self.__derived_sigmoid

    def init_params(self):
        self.P = len(self.aug_x)
        self.error = 0
        self.p = 1
        self.k = 1
        self.c = 1 # learning rate
        self.W = np.random.random((self.aug_x.shape[-1]))

    def augment_input(self, x_input):
        # Note: Augmentation is bias. If augmented by 1, then while calculating net value, the bias multiplier is 
        # part of the weights matrix.
        _input = []
        for i in x_input:
            _input.append(np.concatenate((i, [1])))
        return np.array(_input)

    def set_weights(self, W):
        self.W = np.array(W)

    def __sigmoid(self, net):
        '''
        Returns bipolar activation value for the net
        '''
        return (1 - np.exp(-self._lambda * net)) / (1 + np.exp(-self._lambda * net))

    def __derived_sigmoid(self, output):
        '''
        returns the value of the derived sigmoid. This is a simplified version of the 
        dervative of the original sigmoid curve.
        '''
        return (1 - output ** 2) / 2

    def train(self, interactive=False):
        stop = False
        while not stop:
            if interactive:
                print('Training Cycle #{}:'.format(self.k))
            self.E = 0

            # Training cycle begins
            for step in range(self.P):
                if interactive:
                    print('Step #{}:'.format(self.p))
                    print('%20s : %s' % ('W (before adjust)', (self.W)))
                    print('%20s : %s' % ('Input', (self.aug_x[step])))
                
                # Calculate net for pth pattern
                net = np.dot(self.W.T, self.aug_x[step])
                if net == 0:
                    if self.y_input[step] == -1:
                        output = 1
                    elif self.y_input[step] == 1:
                        output = -1
                    # output = 0
                else:
                    output = self.__activation(net)

                # Calculate the weight change based on gradient descent technique
                _delta = self.eta * 0.5 * (self.y_input[step] - output) * self.aug_x[step] * self.__dev_activation(output)
                self.W = self.W + (_delta)
                self.E = self.E + 0.5 * (self.y_input[step] - output) ** 2
                if interactive:
                    print('%20s : %s' % ('Outputs (d, O)', (self.y_input[step], output)))
                    print('%20s : %s' % ('Error', self.E))
                    print('%20s : %s' % ('Weight change', _delta))
                    print('%20s : %s' % ('W (after adjust)', self.W))
                self.p += 1

            self.cycle_wise_errors.append(self.E)
            if self.E < self.e_max:
                stop = True
            self.k += 1
            if interactive:
                input()

        print('Training steps required: {}'.format(self.p))

if __name__=='__main__':
    # Testing for AND
    x = [[0, 0],
         [0, 1],
         [1, 0],
         [1, 1]]

    y = [-1, 1, 1, 1]

    x = np.array(x)
    y = np.array(y)
    s = ContinuousPerceptron(x, y, learning_rate=1, e_max=0.001)
    s.init_params()
    s.train(interactive=True)