xai_img6.py

# focus on 14th layer of the CNN model
print(model.layers[14].input)
Out [49]: <tf.Tensor 'block4_conv3_2/Relu:0' shape=(?, 28, 28, 512) dtype=float32>

# make model predictions
e = shap.GradientExplainer((model.layers[14].input, model.layers[-1].output), 
                            map2layer(preprocess_input(X.copy()), 14))
shap_values, indexes = e.shap_values(map2layer(to_predict, 14), ranked_outputs=2)
index_names = np.vectorize(lambda x: class_names[str(x)][1])(indexes)

# visualize model decisions
visualize_model_decisions(shap_values=shap_values, x=to_predict, 
                          labels=index_names, figsize=(20, 40))