esmitt

from sklearn.metrics import roc_auc_score, roc_curve

def plot_roc(name: str, labels: numpy.ndarray, predictions: numpy.ndarray, **kwargs) -> ():
  fp, tp, _ = roc_curve(labels, predictions)
  auc_roc = roc_auc_score(labels, predictions)
  plt.plot(100*fp, 100*tp, label=name + " (" + str(round(auc_roc, 3)) + ")", 
           linewidth=2, **kwargs)
  plt.xlabel('False positives [%]')
  plt.ylabel('True positives [%]')
  plt.title('ROC curve')

esmitt

precision = Precision()
precision.update_state(y_train, y_train_pred)
precision.result().numpy()

esmitt

from sklearn.metrics import confusion_matrix
import seaborn as sns

# notice the threshold
def plot_cm(labels: numpy.ndarray, predictions: numpy.ndarray, p: float=0.5) -> ():
    cm = confusion_matrix(labels, predictions > p)
    # you can normalize the confusion matrix

    plt.figure(figsize=(5,5))
    sns.heatmap(cm, annot=True, fmt="d")

esmitt

# Evaluate the model on the test data using `evaluate`
print("Evaluate on test data")
score_test = model.evaluate(test_ds.batch(batch_size))
for name, value in zip(model.metrics_names, score_test):
    print(name, ': ', value)

esmitt

def plot_metrics(history: History) -> ():
    metrics =  ['loss', 'precision', 'recall', 'auc', 'tp', 'sensitivity']
    for n, metric in enumerate(metrics):
        name = metric.replace("_"," ").capitalize()
        plt.subplot(3, 2, n+1)  # adjust according to metrics
        plt.plot(history.epoch,  history.history[metric], color=colors[0], label='Train')
        plt.plot(history.epoch, history.history['val_'+metric],
                 color=colors[0], linestyle="--", label='Val')
        plt.xlabel('Epoch')
        plt.ylabel(name)

esmitt

import matplotlib.pyplot as plt
from matplotlib import rcParams

rcParams['figure.figsize'] = (12, 10)
colors = plt.rcParams['axes.prop_cycle'].by_key()['color']

def plot_log_loss(history: History, title_label: str, n: int) -> ():
    # Use a log scale to show the wide range of values.
    plt.semilogy(history.epoch,  history.history['loss'],
               color=colors[n], label='Train '+title_label)

esmitt

batch_size = 64

"""
Training the model for 60 epochs using our dataset.
The batch size (64) is the same for the validation data.
Only 1 callback was used, but could be more like TensorBoard, ModelCheckpoint, etc.
"""
history = model.fit(train_ds.batch(batch_size=batch_size),
                    epochs=60,
                    validation_data=validation_ds.batch(batch_size=batch_size),

esmitt

from tensorflow.keras.callbacks import EarlyStopping

"""
This callback will stop the training when there is no improvement in the validation accuracy across epochs
"""
early_callback = EarlyStopping(monitor='val_auc', 
                               verbose=1,
                               patience=10,
                               mode='max',
                               restore_best_weights=True)

esmitt

from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.metrics import TruePositives, FalsePositives, TrueNegatives, FalseNegatives, BinaryAccuracy, Precision, Recall, AUC
from tensorflow.keras.metrics import SpecificityAtSensitivity

"""
Definition of metrics commonly used on medical imaging classification, segmentation, and localization problems.
The metrics will appear on each iteration of the training process to monitor the progress of our design.
"""
METRICS = [

esmitt

from tensorflow.keras.utils import plot_model

plot_model(model, 'my-CNNmodel.png', show_shapes=True)