overfit.py

from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression, Lasso
from sklearn.pipeline import Pipeline
import numpy as np
import matplotlib.pyplot as plt

modelF = lambda deg: Pipeline([
    ('poly', PolynomialFeatures(degree=deg)),
    ('linear', LinearRegression(fit_intercept=False))])

def f(x):
    return 2 + 3*x - x**2

X = (np.random.rand(50) * 10 - 5)
Y = f(X) + np.random.normal(0, 5, size=X.shape)
plt.plot(X, Y, 'x')

plt.plot(np.arange(-5, 5, 0.01), f(np.arange(-5, 5, 0.01)), label='true')

poor_model = modelF(2)
rich_model = modelF(20)

poor_model = poor_model.fit(X[:, np.newaxis], Y)
rich_model = rich_model.fit(X[:, np.newaxis], Y)

def plot_model(model, name):
    X = np.arange(-5, 5, 0.01)
    eY = model.predict(X[:, np.newaxis])
    plt.plot(X, eY, '-', label=name)

def likelihood(model, X, y):
    ey = model.predict(X[:, np.newaxis])
    rss = np.linalg.norm(y - ey)**2
    n = y.size
    return -n/2 * (np.log(2*np.pi) + 1 + np.log(rss/n))

print('poor model log-likelihood: %f' % likelihood(poor_model, X, Y))
print('rich model log-likelihood: %f' % likelihood(rich_model, X, Y))

plot_model(poor_model, 'poor')
plot_model(rich_model, 'rich')
plt.ylim(-40, 15)
plt.xlim(-5, 5)
plt.legend(loc='lower right')
plt.savefig('plot.png')
plt.show()

plot.png