sgd_atom.py

def stochastic_linear_gradient_descent(X, y, theta, alpha, tolerance):
    converged = False
    prev_cost = 0
    n_samples = X.shape[0]
    while not converged:
        # use xrange so you dont have to generate the entire range of numbers
        for i in xrange(n_samples):
            # you have to sample x and y together
            # FYI, using one example at a time can be super slow.
            # you might want to consider using batches which should converge quicker
            sample_x, sample_y = X[i,:], y[i,:]
            hypothesis = np.dot(sample_x, theta)
            loss = hypothesis - sample_y # not sure why you need this
            
            # IMHO, it's better to have this function return the cost and gradient
            # simply because the grad is a partial derivative of the cost w.r.t theta
            curr_cost, grad = compute_lin_cost(theta, sample_x, sample_y)
            theta -= alpha * grad
            
            # this is wayyyy cleaner that if statements :P
            converged = abs(curr_cost - prev_cost) < tolerance
            prev_cost = curr_cost
            print("cost " , curr_cost)
    return theta