Classifier Calibration -- A Minimal Example

classifier-calibration.py

import numpy as np
from sklearn.datasets import make_classification
from sklearn.model_selection import cross_val_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.calibration import CalibratedClassifierCV


X, y = make_classification(
    n_samples=1000, n_features=5, n_redundant=2, 
    n_clusters_per_class=1, random_state=123)

model = DecisionTreeClassifier(random_state=123)
scores = cross_val_score(model, X, y, scoring='roc_auc', cv=10, n_jobs=-1)
print(f'Orig ROC AUC: {np.mean(scores):.3f}')

calibrated = CalibratedClassifierCV(model, method='isotonic', cv=10)

scores = cross_val_score(calibrated, X, y, scoring='roc_auc', cv=10, n_jobs=-1)
print(f'Calibrated ROC AUC: {np.mean(scores):.3f}')