Evaluate ECE

eval_ece.py

def evaluate_ECE(data_iter, net, device, M=15):
    net.eval()

    bins, n = [0.0 for _ in range(M)], 0

    with torch.no_grad():
        for X_batch, y_batch in data_iter:
            p_batch = torch.exp(lsoftmax(net(X_batch.to(device))))
            y_hat_batch = net(X_batch.to(device)).argmax(dim=1)
            for p, y_hat, y in zip(p_batch.numpy(), y_hat_batch.numpy(), y_batch.numpy()):
                p[p > 1.0] = 1.0
                p[p < 0.0] = 0.0
                idx = math.floor(p[y_hat] * M)
                if idx >= M:
                    idx = M - 1
                bins[idx] += float(y_hat == y) - p[y_hat]
            n += y_batch.shape[0]
    
    return sum(map(abs, bins)) / n