See https://github.com/andrewschreiber/numpy-saliency

main.py

from model.data import mnist_train_test_sets
from model.network import LeNet5
from saliency.vanilla_gradient import save_vanilla_gradient
# Get MNIST dataset, preprocessed
train_images, train_labels, test_images, test_labels = mnist_train_test_sets()

# Load net with 98% acc weights
net = LeNet5(weights_path="15epoch_weights.pkl")

# Generate saliency maps for the first 10 images
save_vanilla_gradient(network=net, data=train_images[:10], labels=target_labels[:10])