R: SVM

SimpleSVM.R

set.seed(1026)
# dummy 2-D data: yi~xi(x1,x2)
x = rbind(cbind(rnorm(50, 1, 0.3), rnorm(50, 2, 0.3)), cbind(rnorm(50, 2, 0.3), rnorm(50, 1, 0.3)))
#           [,1]     [,2]
# [1,] 1.6565944 1.696906
# [2,] 0.9467358 1.458583
# [3,] 0.9444174 1.513886
# [4,] 0.2480391 2.273355
# [5,] 0.8328066 2.078371
# [6,] 0.9569322 2.174178

# dummy tags
y = c(rep(1, 50), rep(-1, 50))

h <- function(w, C = 1){
  b = w[length(w)]
  w = w[-length(w)]

  # Discrimination function
  # g = y * (x1*w1 + x2*w2 + w3) for 2D data
  g = y * (x %*% w + b)


  # soft margin 
  0.5 * w %*% w + C * sum(1 - g[g < 1])
}

par(mfrow = c(2,2))
regulation = c(1, 5, 10, 100)
for(i in 1:4){
  # optim to minimize function
  # start with w(0,0,0). 
  res = optim(c(0, 0, 0), h, C = regulation[i], method = "L-BFGS-B")

  # best parameters
  w = res$par

  # Plot results
  # w1*x1 + w2*x2 + w3 = 1 for positive data, and vice versa
  line.x = seq(min(x[,1]), max(x[,1]), 0.1)
  line.pos.x2 = (1 - line.x * w[1] - w[3]) / w[2]
  line.neg.x2 = (-1 - line.x * w[1] - w[3]) / w[2]

  plot(x, col = y + 2, main = paste("C = ", regulation[i], sep = ""))
  lines(line.x, line.pos.x2, type = "o",col="red")
  lines(line.x, line.neg.x2, type = "o",col="red")
}