running keras/tensorflow on different machine server with local client

client.py

import requests
from timeit import default_timer as timer

PORT = "9999"
PRED_KERAS_REST_API_URL = "http://localhost:"+PORT+"/predict"
TIME_KERAS_REST_API_URL = "http://localhost:"+PORT+"/time"
IMAGE_PATH = "small.jpg"

# load the input image and construct the payload for the request
image = open(IMAGE_PATH, "rb").read()
payload = {"image": image}

# submit the request
start = timer()
r = requests.post(TIME_KERAS_REST_API_URL, files=payload).json()
end = timer()
print("Time", (end-start))
print("request data", r)

start = timer()
r = requests.post(PRED_KERAS_REST_API_URL, files=payload).json()
end = timer()
print("Time", (end-start))
print("request data", r)

commands.py

[TEXT FILE, i just liked the highlighting on this gist]
///////////////////////////////////////////////////////
Going from a testcase of connecting to distant *Jupyter notebook*:
  1.) on server
  jupyter notebook --no-browser --port=8889

  2.) on local
  ssh -N -f -L localhost:8888:localhost:8889 USERNAME@HOST
  (open tunnel, run in bg)

  open web site
  localhost:8888
    
We can use the same for our python server *run_keras_server.py*:
Lets assume that we have started a interactive shell on our server, and so we need two tunnels (one from local to server and one from the server to the interactivelly run machine)
[on server]

interact --gpu --egress
#Load your Anacondas and other snakes ~~~ this is PSC Bridge specific:
#module load keras/2.0.6_anaconda # PSC Bridge specific,
#source activate $KERAS_ENV

python run_keras_server.py
# tells me "Running on http://0.0.0.0:8123/"
# because of line 113: app.run(host='0.0.0.0', port=8123)

# to help see what is the name of the interactive machine call:
squeue -u USERNAME
# lets assume its name is bobmarley
# we can now connect to it via: ssh bobmarley

# lets build some tunnels then...
[on local]
[local 9999 >> server to bobmarley -> our desired gpu machine 8123]
ssh -N -f -L  9999:bobmarley:8123 USERNAME@HOST

You are there! Try with:
web address: http://localhost:9999/predict # should give you "Method Not Allowed"
curl -X POST -F [email protected] 'http://localhost:9999/predict' # imagine a dog.jpg here
or ultimatelly run the client:
  python client.py

"""
Time 0.07783991500036791
request data {'imageshape': [608, 608], 'internal_time': 0.007249436996062286, 'success': True}
Time 0.07160759600083111
request data {'imageshape': [608, 608], 'internal_time': 0.003656726999906823, 'success': True}
Time 0.1039813960014726
request data {'predictions': [{'label': 'cinema', 'probability': 0.3615420460700989}, {'label': 'unicycle', 'probability': 0.22204479575157166}, {'label': 'street_sign', 'probability': 0.05314052477478981}, {'label': 'jinrikisha', 'probability': 0.04694230109453201}, {'label': 'traffic_light', 'probability': 0.04335896298289299}], 'success': True}
"""

run_keras_server.py

# USAGE
# Start the server:
# 	python run_keras_server.py
# Submit a request via cURL:
# 	curl -X POST -F [email protected] 'http://localhost:9999/predict'
# Submita a request via Python:
#	python simple_request.py

# import the necessary packages
from keras.applications import ResNet50
from keras.preprocessing.image import img_to_array
from keras.applications import imagenet_utils
import tensorflow as tf
from PIL import Image
import numpy as np
import flask
import io
from timeit import default_timer as timer


# initialize our Flask application and the Keras model
app = flask.Flask(__name__)
model = None

def load_model():
	# load the pre-trained Keras model (here we are using a model
	# pre-trained on ImageNet and provided by Keras, but you can
	# substitute in your own networks just as easily)
	global model
	model = ResNet50(weights="imagenet")
	global graph ### IMPORTANT FIX TO ERROR MENTIONED AT https://github.com/keras-team/keras/issues/2397
	graph = tf.get_default_graph()

def prepare_image(image, target):
	# if the image mode is not RGB, convert it
	if image.mode != "RGB":
		image = image.convert("RGB")

	# resize the input image and preprocess it
	image = image.resize(target)
	image = img_to_array(image)
	image = np.expand_dims(image, axis=0)
	image = imagenet_utils.preprocess_input(image)

	# return the processed image
	return image

@app.route("/time", methods=["POST"])
def time():
	data = {"success": False}
	start = timer()

	# ensure an image was properly uploaded to our endpoint
	if flask.request.method == "POST":
		if flask.request.files.get("image"):
			image = flask.request.files["image"].read()
			image = Image.open(io.BytesIO(image))

			data["imageshape"] = image.size

			end = timer()
			data["internal_time"] = end - start
			data["success"] = True


	# return the data dictionary as a JSON response
	return flask.jsonify(data)


@app.route("/predict", methods=["POST"])
def predict():
	# initialize the data dictionary that will be returned from the
	# view
	data = {"success": False}

	# ensure an image was properly uploaded to our endpoint
	if flask.request.method == "POST":
		if flask.request.files.get("image"):
			# read the image in PIL format
			image = flask.request.files["image"].read()
			image = Image.open(io.BytesIO(image))

			# preprocess the image and prepare it for classification
			image = prepare_image(image, target=(224, 224))

			with graph.as_default():
				# classify the input image and then initialize the list
				# of predictions to return to the client
				preds = model.predict(image)
				results = imagenet_utils.decode_predictions(preds)
				data["predictions"] = []

				# loop over the results and add them to the list of
				# returned predictions
				for (imagenetID, label, prob) in results[0]:
					r = {"label": label, "probability": float(prob)}
					data["predictions"].append(r)

				# indicate that the request was a success
				data["success"] = True

	# return the data dictionary as a JSON response
	return flask.jsonify(data)


# if this is the main thread of execution first load the model and
# then start the server
if __name__ == "__main__":
	print(("* Loading Keras model and Flask starting server..."
		"please wait until server has fully started"))
	load_model()
	#app.run()
	app.run(host='0.0.0.0', port=8123)

thanks.txt

Thanks to the original tutorial:
https://blog.keras.io/building-a-simple-keras-deep-learning-rest-api.html
(ps: look at the following more advanced ones at https://www.pyimagesearch.com/2018/02/05/deep-learning-production-keras-redis-flask-apache/)

and for fixing an error with Keras running in run_keras_server.py, which is deployed as a webservice using Flask
https://github.com/keras-team/keras/issues/2397
error:
    raise ValueError("Tensor %s is not an element of this graph." % obj)
    ValueError: Tensor Tensor("fc1000/Softmax:0", shape=(?, 1000), dtype=float32) is not an element of this graph.

Thanks to the original tutorial:
https://blog.keras.io/building-a-simple-keras-deep-learning-rest-api.html
(ps: look at the following more advanced ones at https://www.pyimagesearch.com/2018/02/05/deep-learning-production-keras-redis-flask-apache/)

and for fixing an error with Keras running in run_keras_server.py, which is deployed as a webservice using Flask
keras-team/keras#2397
error:
raise ValueError("Tensor %s is not an element of this graph." % obj)
ValueError: Tensor Tensor("fc1000/Softmax:0", shape=(?, 1000), dtype=float32) is not an element of this graph.