Gradient Descent for linear regression demo.

linear_regression_gradient.R

# Generate random values
set.seed(11111)
x <- rnorm(n = 50)
y <- 10 * x + rnorm(n = 50)

# Add intercept
x <- cbind(1,x)

# Assume random null parameters
param <- c(0,0)

# Store count of observations
m <- length(y)

# Calculate Mean Squared Error cost
cost <- function(x, y, param) {mean(((x %*% param)- y) ^ 2)}
grad <- function(x, y, param) {
  gradient <- rep(0, length(param))
  pre_sum <- ((x %*% param) - y)
  for (i in 1:length(param)) {
    # Squared Error = (x - y) ^ 2
    # Squared Error Gradient: 2 * (x - y)
    gradient[i] <- 2 * mean(pre_sum * x[, i])
  }
  return(gradient)
}

# Set learning rate (eta)
eta <- 0.20

# Set number of boosting iterations
iters <- 20

# Perform boosting with real-time plotting
plot(x[, ncol(x)], y, col = "blue", pch = 20, xlab = "x", ylab = "y")
for(i in 1:iters) {
  abline(param[1], param[2], col = rgb(0.8, 0.0, 0.2, alpha = 0.25))
  #cat("[", sprintf("%02d", i), "] Cost: ", sprintf("%10.07f", cost(x, y, param)), "\n", sep = "")
  cat("[", sprintf("%02d", i), "] Cost: ", sprintf("%10.07f", cost(x, y, param)), ", params = ", paste(sprintf("%08.06f", param), collapse = ", "), "\n", sep = "")
  param <- param - eta * grad(x, y, param)
}
abline(param[1], param[2], col = "blue")