Nested Cross validation GridSearch CV

Nested Cross validation GridSearch CV

from sklearn.datasets import load_iris
from sklearn.model_selection import ParameterGrid, StratifiedKFold

iris = load_iris()
param_grid = [{'kernel': ['rbf'],
               'C': [0.01, 1],
               'gamma': [0.1, 1]},
              {'kernel': ['linear'],
               'C': [0.01, 1]}]
print("List of parameter grids:\n{}".format(param_grid))

def nested_cv(X, y, inner_cv, outer_cv, Classifier, parameter_grid):
    outer_scores = []
    for training_samples, test_samples in outer_cv.split(X, y):
        best_params = {}
        best_score = -np.inf
        for parameters in parameter_grid:
            cv_scores = []
            for inner_train, inner_test in inner_cv.split(X[training_samples], y[training_samples]):
                inner_clf = Classifier(**parameters)
                inner_clf.fit(X[inner_train], y[inner_train])
                inner_score = inner_clf.score(X[inner_test], y[inner_test])
                cv_scores.append(inner_score)
            mean_score = np.mean(cv_scores)
            if mean_score > best_score:
                best_score = mean_score
                best_params = parameters
        outer_clf = Classifier(**best_params)
        outer_clf.fit(X[training_samples], y[training_samples])
        outer_score = outer_clf.score(X[test_samples], y[test_samples])
        outer_scores.append(outer_score)
    return np.array(outer_scores)

scores = nested_cv(iris.data, iris.target, StratifiedKFold(5), StratifiedKFold(5), SVC, ParameterGrid(param_grid))
print('Cross validation scores: {}'.format(scores))
print('Cross validation mean: {}'.format(scores.mean()))