show_linear_sigmoid_sgd.py

import numpy as np

# logistic regression.
# yi = sigmoid(dot(a * xi))

def sigmoid(x):
  return 1 / (1 + np.exp(-x))

N = 1000
M = 2
A = np.random.random((N, M))
x = np.random.random(M)
noise = np.random.random(N) * 0.001
y = np.dot(A, x) + noise
c = sigmoid(y)

# start optimization with cross-entropy loss
# see http://peterroelants.github.io/posts/neural_network_implementation_part02/
l2_reg = 0.001 # l2 regularization term
r = 0.1 # learning rate
e = 0   # error term
w = np.zeros(M)
for i in range(0, N):
  a = A[i, :]  # current sample
  # feed forward
  yy = np.dot(a, w)
  cc = sigmoid(yy)
  e = cc - c[i]
  print(e)
  # backprop
  g = e * a + l2_reg * w # final gradient
  w = w - r * g
  
print(x)
print(w)