A minimal working example of how to implement backpropagation having only NumPy.

simple_backpropagation_in_plain_numpy.py

import numpy as np


class TwoLayerPerceptron(object):
    """
    This is a simple neural network
    with exactly one hidden layer
    and 0.5 * MSE (Mean Squared Error)
    as loss.

    Available hyperparameters are as
    follows: number of features, number
    of hidden units, number of targets,
    and learning rate.
    """

    def __init__(self, n_input_units, n_hidden_units, n_output_units,
                 learning_rate):
        """
        @type n_input_units: int
        @type n_hidden_units: int
        @type n_output_units: int
        @type learning_rate: float
        """

        def sigmoid(x):
            """
            Sigmoid activation function.

            @type x: numpy.ndarray
            @rtype: numpy.ndarray
            """
            return 1 / (1 + np.exp(-x))

        self.n_input_units = n_input_units
        self.n_hidden_units = n_hidden_units
        self.n_output_units = n_output_units

        self.learning_rate = learning_rate
        self.activation_function = sigmoid

        self.weights_input_to_hidden = np.random.normal(
            0.0, self.n_hidden_units**-0.5,
            (self.n_hidden_units, self.n_input_units))
        self.weights_hidden_to_output = np.random.normal(
            0.0, self.n_output_units**-0.5,
            (self.n_output_units, self.n_hidden_units))

        self.updates_input_to_hidden = None
        self.updates_hidden_to_output = None

    def __reset_updates_of_weights(self):
        """
        Resets updates of weights to
        zero vectors.

        @rtype: NoneType
        """
        self.updates_input_to_hidden = np.zeros_like(
            self.weights_input_to_hidden)
        self.updates_hidden_to_output = np.zeros_like(
            self.weights_hidden_to_output)

    def __forward_pass(self, inputs_list, all_layers=False):
        """
        Finds activations for object that is
        represented by `inputs_list`.

        @type inputs_list: list-like
        @type all_layers: bool
        @rtype: numpy.ndarray
                or tuple(numpy.ndarray)
        """
        # To be sure that it is 2-dimensional.
        inputs = np.array(inputs_list, ndmin=2).T

        hidden_inputs = inputs
        hidden_outputs = self.activation_function(
            np.dot(self.weights_input_to_hidden, hidden_inputs))

        final_inputs = hidden_outputs
        final_outputs = np.dot(self.weights_hidden_to_output, final_inputs)

        if all_layers:
            return final_outputs, hidden_outputs, inputs
        else:
            return final_outputs

    def __backward_pass(self, targets_list, final_outputs,
                        hidden_outputs, hidden_inputs):
        """
        Computes updates of weights on one object
        by backpropagation of errors.

        Here partial derivatives of loss
        with respect to values of neurons
        before activation are called deltas.

        @type targets_list: list-like
        @type final_outputs: numpy.ndarray
        @type hidden_outputs: numpy.ndarray
        @type: hidden_inputs: numpy.ndarray
        @rtype: NoneType
        """
        targets = np.array(targets_list, ndmin=2).T

        output_errors = targets - final_outputs
        output_activation_derivative = 1  # Because there is no activation
        output_delta = output_errors * output_activation_derivative

        hidden_propagated = np.dot(self.weights_hidden_to_output.T,
                                   output_delta)
        hidden_activation_derivative = hidden_outputs * (1 - hidden_outputs)
        hidden_delta = hidden_propagated * hidden_activation_derivative

        self.updates_hidden_to_output += self.learning_rate * \
                                         np.dot(output_delta, hidden_outputs.T)
        self.updates_input_to_hidden += self.learning_rate * \
                                        np.dot(hidden_delta, hidden_inputs.T)

    def train(self, batch_inputs, batch_targets):
        """
        Trains neural network on a batch
        with features given by `batch_inputs`
        and target values stored in
        `batch_targets`.

        @type batch_inputs: list-like(list-like)
        @type batch_targets: list-like(list-like)
        @rtype: NoneType
        """
        self.__reset_updates_of_weights()
        for inputs_list, targets_list in zip(batch_inputs, batch_targets):
            all_layers_outputs = self.__forward_pass(inputs_list,
                                                     all_layers=True)
            self.__backward_pass(targets_list, *all_layers_outputs)
        self.weights_hidden_to_output += self.updates_hidden_to_output
        self.weights_input_to_hidden += self.updates_input_to_hidden

    def predict(self, objects_list):
        """
        Makes predictions for `objects_list`
        where rows represent objects and
        columns represent features.

        @type objects_list: list-like(list-like)
        @rtype: numpy.ndarray
        """
        return np.array([self.__forward_pass(inputs_list)
                         for inputs_list in objects_list])