lasso_is_great.py

from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression, Lasso, Ridge
from sklearn.pipeline import Pipeline
import numpy as np
import matplotlib.pyplot as plt

def f(x):
    return 2 + 3*x - x**2

X = (np.random.rand(80) * 10 - 5)
Y = f(X) + np.random.normal(0, 5, size=X.shape)
plt.plot(X, Y, 'x')

plt.plot(np.arange(-5, 5, 0.01), f(np.arange(-5, 5, 0.01)), label='true')

degree = 20
idiot_model = Pipeline([
    ('poly', PolynomialFeatures(degree=degree)),
    ('regression', LinearRegression(fit_intercept=False))])
lasso_model = Pipeline([
    ('poly', PolynomialFeatures(degree=degree)),
    ('regression', Lasso(alpha=1.0, fit_intercept=False, tol=0.01, max_iter=10000))])
ridge_model = Pipeline([
    ('poly', PolynomialFeatures(degree=degree)),
    ('regression', Ridge(1, fit_intercept=False))])

def plot_model(model, name):
    X = np.arange(-5, 5, 0.01)
    eY = model.predict(X[:, np.newaxis])
    plt.plot(X, eY, '-', label=name)

def likelihood(model, X, y):
    ey = model.predict(X[:, np.newaxis])
    rss = np.linalg.norm(y - ey)**2
    n = y.size
    return -n/2 * (np.log(2*np.pi) + 1 + np.log(rss/n))

def proc(name, model):
    model = model.fit(X[:, np.newaxis], Y)

    print('%s model log-likelihood: %f' % (name, likelihood(model, X, Y)))

    print(model.get_params()['steps'][-1][1].coef_)
    print('-' * 50)

    plot_model(model, name)

proc('idiot', idiot_model)
proc('lasso', lasso_model)
proc('ridge', ridge_model)

plt.ylim(-40, 15)
plt.xlim(-5, 5)
plt.legend(loc='lower right')
plt.show()