Closed form ridge regression.

closed_form_ridge_regression.R

library(tidyverse)

y <- as.matrix(mtcars$mpg)
X <- model.matrix(mpg ~ ., mtcars)

# linear regression
solve(crossprod(X)) %*% t(X) %*% y
solve(crossprod(X) + 0 * diag(rep(1, ncol(X)))) %*% t(X) %*% y

# ridge regression
ridge <- function(lambda) {
  solve(crossprod(X) + lambda * diag(rep(1, ncol(X)))) %*% t(X) %*% y
}

# visualize regularization of coefficients
seq(0, 1, by = .01) %>%
  set_names() %>%
  map(ridge) %>%
  map(as.data.frame) %>%
  map(~ rename(.x, estimate = V1)) %>%
  map(rownames_to_column) %>%
  enframe() %>%
  unnest() %>%
  mutate(lambda = as.numeric(name)) %>%
  filter(rowname != '(Intercept)') %>% 
  ggplot(aes(estimate, rowname, colour = lambda)) +
  geom_vline(xintercept = 0, linetype = 2) +
  geom_point(alpha = 0.4) +
  hrbrthemes::theme_ipsum_rc()