Neural Network with Backpropagation, Bootstrapping and Bagging (Uses MSE)

app.js

function mse (errors) {
    var sum = 0;
    for (var i = 0; i < errors.length; i++) {
        sum += Math.pow(errors[i], 2);
    }
    return sum / errors.length;
};

function zero_array (size) {
    var ary = [];
    for (var s = 0; s < size; s++) {
        ary.push(0);
    }
    return ary;
};

function random_array_range (size, low, high) {
    var range = high - low;
    var ary = [];
    for (var s = 0; s < size; s++) {
        ary.push((Math.random() * range) + low);
    }
    return ary; 
};

function delta_abs_ary (a, b) {
    var d = zero_array(a.length);
    for (var i = 0; i < b.length; b++) {
        d[i] = Math.abs(b[i] - a[i]);
    }
    return d;
};

function vector_class_match (a, b) {
    var class_a = vector_class(a);
    var class_b = vector_class(b);
    if (class_a == class_b) {
        return 1;
    } else {
        return 0;
    }
};

function vector_class (a) {
    var max_component       = -1;
    var max_component_value = -Infinity;
    for (var i = 0; i < a.length; i++) {
        if (a[i] > max_component_value) {
            max_component = i;
            max_component_value = a[i];
        }
    }
    return max_component;
};

function neural_network (layer_sizes) {
    var nn = {
        layer_sizes:  layer_sizes,
        output_layer: layer_sizes.length - 1,
        biases:       [], 
        weights:      [],
        outputs:      [],
        deltas:       [],
        changes:      [],
        errors:       [],
    };

    nn.run = function (input) {
        nn.outputs[0] = input;
        for (var layer = 1; layer <= nn.output_layer; layer++) {
            for (var node = 0; node < nn.layer_sizes[layer]; node++) {
                // linear weighting 
                var weights = nn.weights[layer - 1][node];
                var sum     = nn.biases[layer - 1][node];
                for (var i = 0; i < weights.length; i++) {
                    sum += weights[i] * input[i];
                }
                // activation function
                nn.outputs[layer][node] = 1 / (1 + Math.exp(-sum));
            }
            input = nn.outputs[layer];
        }
        return input;
    };

    nn.errors_fitness = function (data_rows) {
        var errors = [];
        for (var r = 0; r < data_rows.length; r++) {
            var estimate = nn.run(data_rows[r][0]);
            var actual   = data_rows[r][1];
            var delta = delta_abs_ary(estimate, actual);
            errors.push(delta);
        }
        return mse(errors);
    };

    nn.errors_classify = function (data_rows) {
        var correct = 0;
        for (var r = 0; r < data_rows.length; r++) {
            var estimate = nn.run(data_rows[r][0]);
            var actual   = data_rows[r][1];
            correct += vector_class_match(estimate, actual);
        }
        return 1.0 - (correct / data_rows.length);
    };

    nn.test = function (data_rows) {
        var outputs = [];
        for (var r = 0; r < data_rows.length; r++) {
            var estimate = nn.run(data_rows[r][0]);
            outputs.push(estimate);
        }
        return outputs;
    };

    nn.test_classify = function (data_rows) {
        var outputs = [];
        for (var r = 0; r < data_rows.length; r++) {
            var estimate = nn.run(data_rows[r][0]);
            outputs.push(vector_class(estimate));
        }
        return outputs;
    };

    nn.train = function (data_rows, cfg) {
        cfg.iterations         = cfg.iterations         || 20000;
        cfg.error_threshold    = cfg.error_threshold    || 0.005;
        cfg.learning_rate      = cfg.learning_rate      || 0.3;
        cfg.verbose            = cfg.verbose            || true;
        cfg.verbose_log_period = cfg.verbose_log_period || 10;

        var last_error = null;
        var error      = 1;
        var iteration  = 0;
        for (iteration = 0; iteration < cfg.iterations && error > cfg.error_threshold && error != last_error; iteration++) {
            last_error = error;
            var sum = 0;
            for (var r = 0; r < data_rows.length; r++) {
                sum += nn.train_pattern(data_rows[r][0], data_rows[r][1], cfg);
            }   
            error = sum / data_rows.length;
            if (cfg.verbose && (iteration % cfg.verbose_log_period == 0)) {
                console.log((cfg.verbose_network_index != undefined ? 'network: ' + cfg.verbose_network_index + '/' + cfg.verbose_network_count + ', ' : '') + 'iterations: ' + iteration + '/' + cfg.iterations +', training error: ' + error + '/' + cfg.error_threshold);
            }
        }
        return {
            error:      error,
            iterations: iteration
        };
    };

    nn.train_pattern = function (input, output, cfg) {
        nn.run(input);
        nn.calculate_deltas(output);
        nn.adjust_weights(cfg.learning_rate, cfg.momentum);
        return mse(nn.errors[nn.output_layer]);
    };

    nn.calculate_deltas = function (output) {
        for (var layer = nn.output_layer; layer >= 0; layer--) {
            for (var node = 0; node < nn.layer_sizes[layer]; node++) {
                var nn_output = nn.outputs[layer][node];
                var error = 0;
                if (layer == nn.output_layer) {
                    error = output[node] - nn_output;
                }
                else {
                    var deltas = nn.deltas[layer + 1];
                    for (var i = 0; i < deltas.length; i++) {
                        error += deltas[i] * nn.weights[layer][i][node];
                    }
                }
                nn.errors[layer][node] = error;
                nn.deltas[layer][node] = error * nn_output * (1 - nn_output);
            }
        }
    };

    nn.adjust_weights = function (learning_rate, momentum) {
        for (var layer = 1; layer <= nn.output_layer; layer++) {
            var incoming = nn.outputs[layer - 1];
            for (var node = 0; node < nn.layer_sizes[layer]; node++) {
                var delta = nn.deltas[layer][node];
                for (var i = 0; i < incoming.length; i++) {
                    var change = nn.changes[layer - 1][node][i];
                    change = (learning_rate * delta * incoming[i]) + (momentum * change);
                    nn.changes[layer - 1][node][i] = change;
                    nn.weights[layer - 1][node][i] += change;
                }
                nn.biases[layer - 1][node] += learning_rate * delta;
            }
        }
    };

    for (var layer = 0; layer <= nn.output_layer; layer++) {
        var layer_size = nn.layer_sizes[layer];
        nn.deltas.push(zero_array(layer_size));
        nn.errors.push(zero_array(layer_size));
        nn.outputs.push(zero_array(layer_size));
        if (layer > 0) {
            nn.biases.push(random_array_range(layer_size, -0.2, 0.2));
            nn.weights.push([]);
            nn.changes.push([]);
            for (var node = 0; node < layer_size; node++) {
                var previous_layer_size = nn.layer_sizes[layer - 1];
                nn.weights[nn.weights.length - 1].push(random_array_range(previous_layer_size, -0.2, 0.2));
                nn.changes[nn.changes.length - 1].push(zero_array(previous_layer_size));
            }
        }
    }

    return nn;
};

function weight_drop_neural_network (layer_sizes) {
    var wdnn = neural_network(layer_sizes);

    wdnn.run_with_weight_drop = function (input, drop_rate) {
        wdnn.outputs[0] = input;
        for (var layer = 1; layer <= wdnn.output_layer; layer++) {
            for (var node = 0; node < wdnn.layer_sizes[layer]; node++) {
                // linear weighting 
                var weights = wdnn.weights[layer - 1][node];
                var sum     = wdnn.biases[layer - 1][node];
                for (var i = 0; i < weights.length; i++) {
                    sum += (weights[i] * input[i]) * (Math.random() < drop_rate ? 0 : 1);
                }
                // activation function
                wdnn.outputs[layer][node] = 1 / (1 + Math.exp(-sum));
            }
            input = wdnn.outputs[layer];
        }
        return input;
    };

    wdnn.train_pattern = function (input, output, cfg) {
        wdnn.run_with_weight_drop(input, cfg.drop_rate);
        wdnn.calculate_deltas(output);
        wdnn.adjust_weights(cfg.learning_rate, cfg.momentum);
        return mse(wdnn.errors[wdnn.output_layer]);
    };

    return wdnn;
};

function bagged_neural_network (neural_network_count, layer_sizes, fn_neural_network) {
    var bnn = {
        neural_network_count: neural_network_count,
        layer_sizes:          layer_sizes,
        output_layer:         layer_sizes.length - 1,
        networks:             []
    };
    for (var n = 0; n < neural_network_count; n++) {
        bnn.networks.push(fn_neural_network(layer_sizes));
    }

    bnn.run = function (input) {
        var output_sum = zero_array(bnn.layer_sizes[bnn.output_layer]);
        for (var n = 0; n < bnn.networks.length; n++) {
            var output = bnn.networks[n].run(input);
            for (var o = 0; o < output.length; o++) {
                output_sum[o] += output[o];
            }
        }
        for (var o = 0; o < output_sum.length; o++) {
            output_sum[o] /= bnn.neural_network_count;
        }
        return output_sum;
    };

    bnn.errors_fitness = function (data_rows) {
        var errors = [];
        for (var r = 0; r < data_rows.length; r++) {
            var estimate = bnn.run(data_rows[r][0]);
            var actual   = data_rows[r][1];
            var delta = delta_abs_ary(estimate, actual);
            errors.push(delta);
        }
        return mse(errors);
    };

    bnn.errors_classify = function (data_rows) {
        var correct = 0;
        for (var r = 0; r < data_rows.length; r++) {
            var estimate = bnn.run(data_rows[r][0]);
            var actual   = data_rows[r][1];
            correct += vector_class_match(estimate, actual);
        }
        return 1.0 - (correct / data_rows.length);
    };

    bnn.test = function (data_rows) {
        var outputs = [];
        for (var r = 0; r < data_rows.length; r++) {
            var estimate = bnn.run(data_rows[r][0]);
            outputs.push(estimate);
        }
        return outputs;
    };

    bnn.test_classify = function (data_rows) {
        var outputs = [];
        for (var r = 0; r < data_rows.length; r++) {
            var estimate = bnn.run(data_rows[r][0]);
            outputs.push(vector_class(estimate));
        }
        return outputs;
    };

    bnn.train = function (data_rows, cfg) {
        var errors = [];
        cfg.verbose_network_count = bnn.networks.length;
        for (var n = 0; n < bnn.networks.length; n++) {
            cfg.verbose_network_index = n;
            errors.push(bnn.networks[n].train(data_rows, cfg));
        }
        return errors;
    };

    return bnn
};

function bootstrapped_neural_network (neural_network_count, layer_sizes, fn_neural_network) {
    var bnn = bagged_neural_network(neural_network_count, layer_sizes, fn_neural_network);

    bnn.train = function (data_rows, cfg) {
        var errors = [];
        cfg.verbose_network_count = bnn.networks.length;
        for (var n = 0; n < bnn.networks.length; n++) {
            var samples = [];
            cfg.verbose_network_index = n;
            for (var s = 0; s < cfg.samples_per_network; s++) {
                samples.push(data_rows[Math.floor(Math.random() * data_rows.length)]);
            }
            errors.push(bnn.networks[n].train(samples, cfg));
        }
        return errors;
    };

    return bnn
};

var ocr_train         = require('./data/ocr/train.js');
var ocr_test          = require('./data/ocr/test.js');
var ocr_write_results = require('./data/ocr/write_results.js');

console.log(ocr_train[0][0].length, ocr_train[0][1].length);
console.log(ocr_test[0][0].length, ocr_test.length);


var bnn = bootstrapped_neural_network(10, [784, 100, 10], weight_drop_neural_network);
var cfg_train = {
    drop_rate:           0.5,
    samples_per_network: 500,
    iterations:          100, 
    error_threshold:     0.005, 
    learning_rate:       0.3, 
    momentum:            0.1, 
    verbose:             true, 
    verbose_log_period:  5
};
var errors = bnn.train(ocr_train, cfg_train);
console.log('network errors', errors);
console.log('accuracy', bnn.errors_classify(ocr_train));

var test_answers = bnn.test_classify(ocr_test);
ocr_write_results(test_answers);