Skip to content

Instantly share code, notes, and snippets.

@amintos

amintos/nn.py

Last active October 13, 2015 02:58
Show Gist options
  • Save amintos/4128479 to your computer and use it in GitHub Desktop.
Save amintos/4128479 to your computer and use it in GitHub Desktop.
Download ZIP
standalone neural networks for simple function approximation (fixed SingleLayerPerceptron and added Biases, 2013/12/05)
Raw
nn.py
#
# NEURAL NETWORK CLASSIFIERS
# Pure Python Implementation
#
# Copyright (c) 2008 - 2013 | Toni Mattis <[email protected]>
#
import random, math
# ------------------------------------------------------------------------------
# SINGLE LAYER PERCEPTRON
# (1 passive input layer, ) 1 output layer
# ------------------------------------------------------------------------------
class SingleLayerPerceptron(object):
def __init__(self, n_input, n_output, init_dev = 10.0):
self.n_input = n_input
self.n_output = n_output
self.m = [ [random.random() * init_dev - init_dev / 2
for h in xrange(n_output)]
for i in xrange(n_input + 1) ]
self.input_layer = [0.] * n_input + [1.]
self.output_layer = [0.] * n_output
self.actfunc = lambda x: 1. / (1. + math.exp(-x))
self.gradfunc = lambda y: y * (1 - y)
self.rate = 0.3
def classify(self, vector):
assert len(vector) == self.n_input
m = self.m
act = self.actfunc
ni = self.n_input
no = self.n_output
self.input_layer = inp = map(float, vector) + [1.]
self.output_layer = out = \
[act(sum([inp[i] * m[i][h]
for i in xrange(ni + 1)])) for h in xrange(no)]
return out
def feedback(self, error_vector):
assert len(error_vector) == self.n_output
m = self.m
grd = self.gradfunc
ni = self.n_input
no = self.n_output
out = self.output_layer
inp = self.input_layer
rate = self.rate
m = [[m[j][k] + rate * error_vector[k] * grd(out[k]) * inp[j]
for k in xrange(no)]
for j in xrange(ni + 1)]
self.m = m
def learn(self, in_vector, ref_vector):
out_vector = self.classify(in_vector)
err_vector = [ref_vector[i] - out_vector[i]
for i in xrange(self.n_output)]
self.feedback(err_vector)
def learn_set(self, training_set, iterations):
for i in xrange(iterations):
for inp, out in training_set:
self.learn(inp, out)
def mean_error(self, reference_set):
err = 0
for inp, ref in reference_set:
out = self.classify(inp)
err += sum([(ref[i] - out[i]) ** 2 for i in xrange(len(ref))]) /\
len(ref)
return err / len(reference_set)
# ------------------------------------------------------------------------------
# DOUBLE LAYER PERCEPTRON
# (1 passive input layer, ) 1 hidden layer, 1 output layer
# ------------------------------------------------------------------------------
class DoubleLayerPerceptron(object):
'''I'm a non-linear feed-forward neural network with one hidden layer.'''
def __init__(self, n_input, n_hidden, n_output, init_dev = 10.0):
self.n_input = n_input
self.n_hidden = n_hidden
self.n_output = n_output
# synaptic matrices
self.m1 = [ [random.random() * init_dev - init_dev / 2
for h in xrange(n_hidden)]
for i in xrange(n_input + 1) ]
self.m2 = [ [random.random() * init_dev - init_dev / 2
for h in xrange(n_output)]
for i in xrange(n_hidden + 1) ]
# activation vectors for each layer
self.input_layer = [0.] * n_input + [1.]
self.hidden_layer = [0.] * n_hidden + [1.]
self.output_layer = [0.] * n_output
# standard activation and gradient function
#self.actfunc = lambda x: math.tanh(x)
#self.gradfunc = lambda y: 1 - y * y
self.actfunc = lambda x: 1. / (1. + math.exp(-x))
self.gradfunc = lambda y: y * (1 - y)
# learning rate
self.rate = 0.1
def classify(self, vector):
assert len(vector) == self.n_input
# speed-up, minimizes object-attribute lookups in loops
m1 = self.m1
m2 = self.m2
act = self.actfunc
ni = self.n_input
nh = self.n_hidden
no = self.n_output
self.input_layer = inp = map(float, vector) + [1.]
# feed-forward-steps
# implemented as list-comprehension based
# matrix multiplication with instant application of the
# activation function
self.hidden_layer = hid = \
[act(sum([inp[i] * m1[i][h]
for i in xrange(ni + 1)])) for h in xrange(nh)] + [1.]
self.output_layer = out = \
[act(sum([hid[h] * m2[h][o]
for h in xrange(nh + 1)])) for o in xrange(no)]
return out
def backpropagate(self, error_vector):
assert len(error_vector) == self.n_output
m1 = self.m1
m2 = self.m2
grd = self.gradfunc
ni = self.n_input
nh = self.n_hidden
no = self.n_output
out = self.output_layer
hid = self.hidden_layer
inp = self.input_layer
rate = self.rate
# compute gradients for each layer
d_out = [error_vector[k] * grd(out[k]) for k in xrange(no)]
d_hid = [grd(hid[j]) * sum([d_out[k] * m2[j][k] for k in xrange(no)])
for j in xrange(nh + 1)]
# descend gradients by adjusting weight matrices accordingly
# (instead uptading the existing structure we rebuild the
# matrices completely for better performance)
m2 = [[m2[j][k] + rate * d_out[k] * hid[j]
for k in xrange(no)]
for j in xrange(nh + 1)]
m1 = [[m1[i][j] + rate * d_hid[j] * inp[i]
for j in xrange(nh)]
for i in xrange(ni + 1)]
self.m1 = m1
self.m2 = m2
def learn(self, in_vector, ref_vector):
out_vector = self.classify(in_vector)
err_vector = [ref_vector[i] - out_vector[i]
for i in xrange(self.n_output)]
self.backpropagate(err_vector)
def learn_set(self, training_set, iterations):
for i in xrange(iterations):
for inp, out in training_set:
self.learn(inp, out)
def mean_error(self, reference_set):
err = 0
for inp, ref in reference_set:
out = self.classify(inp)
err += sum([(ref[i] - out[i]) ** 2 for i in xrange(len(ref))]) /\
len(ref)
return err / len(reference_set)
# --- Loading & Saving Files ---
def save(self, filename):
filestream = file(filename, "w")
filestream.write("%s,%s,%s\n" % (
self.n_input,
self.n_hidden,
self.n_output))
for v in self.m1:
filestream.write(','.join(map(str, v)) + '\n')
for v in self.m2:
filestream.write(','.join(map(str, v)) + '\n')
filestream.close()
@staticmethod
def load(filename):
filestream = file(filename, "r")
inp, hid, out = map(int, filestream.readline().split(','))
result = DoubleLayerPerceptron(inp, hid, out)
m1 = []
m2 = []
for i in xrange(inp):
m1.append(map(float, filestream.readline().split(',')))
for i in xrange(hid):
m2.append(map(float, filestream.readline().split(',')))
result.m1 = m1
result.m2 = m2
return result
# ------------------------------------------------------------------------------
# TRIPLE LAYER PERCEPTRON
# (1 passive input layer, ) 2 hidden layers, 1 output layer
# ------------------------------------------------------------------------------
class TripleLayerPerceptron(object):
'''I'm a non-linear feed-forward neural network with 2 hidden layers.'''
def __init__(self, n_input, n_hidden_1, n_hidden_2, n_output,
init_dev = 5.0):
self.n_input = n_input
self.n_hidden_1 = n_hidden_1
self.n_hidden_2 = n_hidden_2
self.n_output = n_output
# synaptic matrices
self.m1 = [ [random.random() * init_dev - init_dev / 2
for h in xrange(n_hidden_1)]
for i in xrange(n_input) ]
self.m2 = [ [random.random() * init_dev - init_dev / 2
for h in xrange(n_hidden_2)]
for i in xrange(n_hidden_1) ]
self.m3 = [ [random.random() * init_dev - init_dev / 2
for h in xrange(n_output)]
for i in xrange(n_hidden_2) ]
# activation vectors for each layer
self.input_layer = [0.] * n_input
self.hidden_layer_1 = [0.] * n_hidden_1
self.hidden_layer_2 = [0.] * n_hidden_2
self.output_layer = [0.] * n_output
# standard activation and gradient function
#self.actfunc = lambda x: math.tanh(x)
#self.gradfunc = lambda y: 1 - y * y
self.actfunc = lambda x: 1. / (1. + math.exp(-x))
self.gradfunc = lambda y: y * (1 - y)
# learning rate
self.rate = 0.3
def classify(self, vector):
assert len(vector) == self.n_input
# speed-up, minimizes object-attribute lookups in loops
m1 = self.m1
m2 = self.m2
m3 = self.m3
act = self.actfunc
ni = self.n_input
nh1 = self.n_hidden_1
nh2 = self.n_hidden_2
no = self.n_output
self.input_layer = inp = map(float, vector)
# feed-forward-steps
# implemented as list-comprehension based
# matrix multiplication with instant application of the
# activation function
self.hidden_layer_1 = hid1 = \
[act(sum([inp[i] * m1[i][h]
for i in xrange(ni)])) for h in xrange(nh1)]
self.hidden_layer_2 = hid2 = \
[act(sum([hid1[i] * m2[i][h]
for i in xrange(nh1)])) for h in xrange(nh2)]
self.output_layer = out = \
[act(sum([hid2[h] * m3[h][o]
for h in xrange(nh2)])) for o in xrange(no)]
return out
def backpropagate(self, error_vector):
assert len(error_vector) == self.n_output
m1 = self.m1
m2 = self.m2
m3 = self.m3
grd = self.gradfunc
ni = self.n_input
nh1 = self.n_hidden_1
nh2 = self.n_hidden_2
no = self.n_output
out = self.output_layer
hid1 = self.hidden_layer_1
hid2 = self.hidden_layer_2
inp = self.input_layer
rate = self.rate
# compute gradients for each layer
d_out = [error_vector[k] * grd(out[k]) for k in xrange(no)]
d_hid2 = [grd(hid2[j]) * sum([d_out[k] * m3[j][k] for k in xrange(no)])
for j in xrange(nh2)]
d_hid1 = [grd(hid1[j]) * sum([d_hid2[k] * m2[j][k] for k in xrange(nh2)])
for j in xrange(nh1)]
# descend gradients by adjusting weight matrices accordingly
# (instead uptading the existing structure we rebuild the
# matrices completely for better performance)
m3 = [[m3[j][k] + rate * d_out[k] * hid2[j]
for k in xrange(no)]
for j in xrange(nh2)]
m2 = [[m2[j][k] + rate * d_hid2[k] * hid1[j]
for k in xrange(nh2)]
for j in xrange(nh1)]
m1 = [[m1[i][j] + rate * d_hid1[j] * inp[i]
for j in xrange(nh1)]
for i in xrange(ni)]
self.m1 = m1
self.m2 = m2
self.m3 = m3
def learn(self, in_vector, ref_vector):
out_vector = self.classify(in_vector)
err_vector = [ref_vector[i] - out_vector[i]
for i in xrange(self.n_output)]
self.backpropagate(err_vector)
def learn_set(self, training_set, iterations):
for i in xrange(iterations):
for inp, out in training_set:
self.learn(inp, out)
def mean_error(self, reference_set):
err = 0
for inp, ref in reference_set:
out = self.classify(inp)
err += sum([(ref[i] - out[i]) ** 2 for i in xrange(len(ref))]) /\
len(ref)
return err / len(reference_set)
# --- Loading & Saving Files ---
def save(self, filename):
filestream = file(filename, "w")
filestream.write("%s,%s,%s,%s\n" % (
self.n_input,
self.n_hidden_1,
self.n_hidden_2,
self.n_output))
for v in self.m1:
filestream.write(','.join(map(str, v)) + '\n')
for v in self.m2:
filestream.write(','.join(map(str, v)) + '\n')
for v in self.m3:
filestream.write(','.join(map(str, v)) + '\n')
filestream.close()
@staticmethod
def load(filename):
filestream = file(filename, "r")
inp, hid1, hid2, out = map(int, filestream.readline().split(','))
result = TripleLayerPerceptron(inp, hid1, hid2, out)
m1 = []
m2 = []
m3 = []
for i in xrange(inp):
m1.append(map(float, filestream.readline().split(',')))
for i in xrange(hid1):
m2.append(map(float, filestream.readline().split(',')))
for i in xrange(hid2):
m3.append(map(float, filestream.readline().split(',')))
result.m1 = m1
result.m2 = m2
result.m3 = m3
return result
def encoder_test(net, iterations = 1000, n = 8, m = 3):
code_set = [ [0.] * i + [1.] + [0.] * (n - i - 1)
for i in xrange(n) ]
training_set = [(c, c) for c in code_set]
net.learn_set(training_set, iterations)
e = net.mean_error(training_set)
return net, training_set, e
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment