Assign before doing any keras operation.

gpu_tf_keras_memory_limit.py

# keras example imports
from keras.models import load_model
 
## extra imports to set GPU options
import tensorflow as tf
from keras import backend as k
 
###################################
# TensorFlow wizardry
config = tf.ConfigProto()
 
# Don't pre-allocate memory; allocate as-needed
config.gpu_options.allow_growth = True
 
# Only allow a total of half the GPU memory to be allocated
config.gpu_options.per_process_gpu_memory_fraction = 0.5
 
# Create a session with the above options specified.
k.tensorflow_backend.set_session(tf.Session(config=config))
###################################
 
# returns a compiled model
# identical to the previous one
model = load_model('my_model.h5')
 
# TODO: classify all the things