Python code for Hacker's guide to Neural Networks - Andrej Karpathy blog

neural_node_backpropagation.py

#Understanding neural networks
#Random approach
def forward_multiply_gate(x,y):
	return x*y

x = -2
y = 3

tweak_amount = 0.01
import random
best_out = -999999999999999999

for var in range(100):
	x_try = x + tweak_amount*(random.uniform(0, 1)*2 -1)
	y_try = y + tweak_amount*(random.uniform(0,1)*2-1)
	out = forward_multiply_gate(x_try,y_try)
	if (out > best_out):
		best_out = out
		best_x = x_try
		best_y = y_try

# print (x_try)
# print (y_try)
# print (best_out)


# strategy approach
# compute dervivative
out = forward_multiply_gate(x,y)
h = 0.001
x_h = x + h
out2 = forward_multiply_gate(x_h,y)
x_derivative = (out2-out)/h
y_h = y + h
out3 = forward_multiply_gate(x,y_h)
y_derivative = (out3-out)/h

# print (x_derivative)
# print (y_derivative)


for var in range(100):
	x_try = x + tweak_amount*x_derivative
	y_try = y + tweak_amount*y_derivative
	out = forward_multiply_gate(x_try,y_try)
	if (out > best_out):
		best_out = out
		best_x = x_try
		best_y = y_try

# print (x_try)
# print (y_try)
# print (best_out)

x_try = x + tweak_amount*y
y_try = y + tweak_amount*x
# print (forward_multiply_gate(x_try,y_try))


def forward_multiply_gate(x,y):
	return x*y

def forward_add_gate(x,y):
	return x+y

def forward_circuit(x,y,z):
	q = forward_add_gate(x,y)
	out = forward_multiply_gate(q,z)
	return out

x = -2
y = 5
z = -4

q = forward_add_gate(x,y)
f = forward_circuit(x,y,z)
derivative_f_wrt_z = q 
derivative_f_wrt_q = z 
derivative_q_wrt_y = 1.0
derivative_q_wrt_x = 1.0
derivative_f_wrt_x = derivative_q_wrt_x * derivative_f_wrt_q
derivative_f_wrt_y = derivative_q_wrt_y * derivative_f_wrt_q
# print derivative_f_wrt_x
# print derivative_f_wrt_y
# print derivative_f_wrt_z
step_zize = 0.01
x = x + step_zize*derivative_f_wrt_x
y = y + step_zize*derivative_f_wrt_y
z = z + step_zize*derivative_f_wrt_z

q = forward_add_gate(x,y)
f = forward_multiply_gate(q,z)

# print(q)
# print(f)

# print forward_circuit(x,y,z)
# print forward_circuit(x,y,z)
# print forward_circuit(x_try,y_try,z_try)

# numerical derivative
x,y,z = -2,5,-4
h = 0.0001
x_derivative = (forward_circuit(x+h,y,z)-forward_circuit(x,y,z))/h
y_derivative = (forward_circuit(x,y+h,z)-forward_circuit(x,y,z))/h
z_derivative = (forward_circuit(x,y,z+h)-forward_circuit(x,y,z))/h
# print x_derivative
# print y_derivative
# print z_derivative
# computing analytical derivative properly has made mme correct the function

import math

def sigmoid(x):
  return 1 / (1 + math.exp(-x))

class unit():
	def __init__(self,value,grad):
		self.value = value
		self.grad = grad

class multiply_gate():
	def forward(self,u0,u1):
		self.u0 = u0
		self.u1 = u1
		self.utop = unit(u0.value*u1.value,0.0)
		return self.utop

	def backward(self):
		self.u0.grad += self.u1.value*self.utop.grad
		self.u1.grad += self.u0.value*self.utop.grad

class add_gate():
	def forward(self,u0,u1):
		self.u0 = u0
		self.u1 = u1
		self.utop = unit(u0.value+u1.value,0.0)
		return self.utop

	def backward(self):
		self.u0.grad += 1*self.utop.grad
		self.u1.grad += 1*self.utop.grad

class sigmoid_gate():
	def forward(self,u0):
		self.u0 = u0
		self.utop = unit(sigmoid(self.u0.value),0.0)
		return self.utop

	def backward(self):
		s = sigmoid(self.u0.value)
		self.u0.grad += s*(1-s)*self.utop.grad

a = unit(1,0)
b = unit(2,0)
c = unit(-3,0)
x = unit(-1,0)
y = unit(3,0)

mulg0 = multiply_gate()
mulg1 = multiply_gate()
addg0 = add_gate()
addg1 = add_gate()
sg0 = sigmoid_gate()


ax = mulg0.forward(a,x)
by = mulg1.forward(b,y)
axpby = addg0.forward(ax,by)
axpbypc = addg1.forward(axpby,c)
s = sg0.forward(axpbypc)

# print s.value
# print s.grad

s.grad = 1.0
sg0.backward()
addg1.backward()
addg0.backward()
mulg1.backward()
mulg0.backward()

print a.grad
print b.grad
print c.grad
print x.grad
print y.grad

step_zize = 0.01
a.value += step_zize*a.grad
b.value += step_zize*b.grad

Python implementation of 2 dimensional neural node with back propagation