server_enc_dec.md

Serving Encoder-Decoder Models using TF Serving

Prithiviraj Damodaran

Step1 : Save Models/Weights

import tensorflow as tf
from tensorflow.keras.models import model_from_json
from tensorflow.keras.models import load_model

'''
Initialize model from JSON and reload weights
1. load json and create model
'''
with open("dec_model_num.json", "rb") as dec_json_file:
    loaded_dec_model_json = dec_json_file.read()
    loaded_dec_model = model_from_json(loaded_dec_model_json)

with open("enc_model_num.json", "rb") as enc_json_file:
    loaded_enc_model_json = enc_json_file.read()
    loaded_enc_model = model_from_json(loaded_enc_model_json)

enc_json_file.close()
dec_json_file.close()

### 2.load weights into new model
loaded_dec_model.load_weights("dec_model_num.h5")
loaded_enc_model.load_weights("enc_model_num.h5")
print("Loaded models from disk")

tf.keras.backend.set_learning_phase(0) # Ignore dropout at inference

'''
Fetch the Keras session and save the model
The signature definition is defined by the input and output tensors
And stored with the default serving key
'''

with tf.keras.backend.get_session() as sess:
    tf.saved_model.simple_save(
        sess,
        './encoder_model/1',
        inputs={'input_image': loaded_enc_model.input},
        outputs={t.name:t for t in loaded_enc_model.outputs})
    tf.saved_model.simple_save(
        sess,
        './decoder_model/1',
        inputs ={t.name:t for t in loaded_dec_model.input},
        outputs={t.name:t for t in loaded_dec_model.outputs})


print("loaded")

Step2 : Optionally, checkout the saved Models/Weights

encoder_model > tree -c
.
├── variables
│   ├── variables.data-00000-of-00001
│   └── variables.index
└── saved_model.pb

Step3 : install TF serve if you already dont have it

apt-get install tensorflow-model-server

Step4 : Get Layer names using CMDLINE for the encoder model

encoder_model > saved_model_cli show --dir ./ --all

MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:

signature_def['serving_default']:
  The given SavedModel SignatureDef contains the following input(s):
    inputs['input_image'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, 23)
        name: input_7:0
  The given SavedModel SignatureDef contains the following output(s):
    outputs['bidirectional_3/concat:0'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, 23, 256)
        name: bidirectional_3/concat:0
    outputs['concatenate_6/concat:0'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, 256)
        name: concatenate_6/concat:0
    outputs['concatenate_7/concat:0'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, 256)
        name: concatenate_7/concat:0
  Method name is: tensorflow/serving/predict

Step5 : Get Layer names using CMDLINE for the decoder model

decoder_model> saved_model_cli show --dir ./ --all

MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:

signature_def['serving_default']:
  The given SavedModel SignatureDef contains the following input(s):
    inputs['inf_decoder_inputs:0'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, -1)
        name: inf_decoder_inputs:0
    inputs['state_input_c:0'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, 256)
        name: state_input_c:0
    inputs['state_input_h:0'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, 256)
        name: state_input_h:0
  The given SavedModel SignatureDef contains the following output(s):
    outputs['dense_7/truediv:0'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, -1, 601)
        name: dense_7/truediv:0
    outputs['lstm_1/while/Exit_2:0'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, 256)
        name: lstm_1/while/Exit_2:0
    outputs['lstm_1/while/Exit_3:0'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, 256)
        name: lstm_1/while/Exit_3:0
  Method name is: tensorflow/serving/predict

Step6 : Run TFServe servers to server encoder and decoder model. These are gRPC services that can be used as REST services

nohup sudo tensorflow_model_server --model_base_path=$(pwd) --rest_api_port=9000 --model_name=encoder_model &

nohup sudo tensorflow_model_server --model_base_path=$(pwd) --rest_api_port=9001 --model_name=decode_model &

Step7 : Run your app code to that calls the TFServe REST services and serve the results.

import requests
import json
import pickle
import sys

text = "please cal"

with open('input_lang.pkl', 'rb') as input1:
    input_lang = pickle.load(input1)
with open('output_lang.pkl', 'rb') as target:
    target_lang = pickle.load(target)
with open('output_lang1.pkl', 'rb') as target1:
    lang_target = pickle.load(target1)


import numpy as np
len_target = 23
len_input = 23

def sentence_to_vector(sentence, lang):
    pre = sentence
    vec = np.zeros(len_input)
    sentence_list = [lang[s] for s in pre.split(' ')]
    for i,w in enumerate(sentence_list):
        vec[i] = w
    return vec

# Given an input string, an encoder model (infenc_model) and a decoder model (infmodel),
def translate(input_sentence):
    sv = sentence_to_vector(input_sentence, input_lang)
    #sv = sv.reshape(1,len(sv))
    output_sentence = ""
    print(sv.shape)
    print(sv)
                  
    payload = {
            "instances":[{"input_image": sv.tolist()}]
    }
    try:
      r = requests.post('http://localhost:9000/v1/models/encoder_model:predict', json=payload)
      r.raise_for_status()
    except requests.exceptions.HTTPError as e:
        print(r)
        print(r.content)
        return "Error: " + str(e)
        #json.loads(r.content)
        #sys.exit(1)
    epred= json.loads(r.content)['predictions']
    emb_out = epred[0]['bidirectional_3/concat:0']
    sh = epred[0]['concatenate_6/concat:0']
    sc = epred[0]['concatenate_7/concat:0']
    #[emb_out, sh, sc] = loaded_enc_model.predict(x=sv)
    print(epred[0].keys())
    i = 0
    start_vec = target_lang["<start>"]
    stop_vec = target_lang["<end>"]

    cur_vec = np.zeros((1))
    cur_vec[0] = start_vec
    cur_word = "<start>"
    output_sentence = ""
    print(cur_vec.shape)

    while cur_word != "<end>" and i < (len_target-1):
        i += 1
        if cur_word != "<start>":
            output_sentence = output_sentence + " " + cur_word
        #x_in = [cur_vec,sh, sc]

        ####
        payload = {
            "instances":[{
                          "inf_decoder_inputs:0": cur_vec.tolist(),
                          "state_input_c:0"     : sh,
                          "state_input_h:0"     : sc
                        }
                        ]
         }


        try:
          r = requests.post('http://localhost:9001/v1/models/decoder_model:predict', json=payload)
          r.raise_for_status()
        except requests.exceptions.HTTPError as e:
          print(r)
          print(r.content)
          return "Error: " + str(e)
        ####
        dpred= json.loads(r.content)['predictions']
        print(dpred[0].keys())
        nvec = dpred[0]['dense_7/truediv:0']
        sh   = dpred[0]['lstm_1/while/Exit_2:0']
        sc   = dpred[0]['lstm_1/while/Exit_3:0']
        #[nvec, sh, sc] = loaded_dec_model.predict(x=x_in)

        cur_vec[0] = np.argmax(nvec[0])
        cur_word = lang_target[cur_vec[0]]

    return output_sentence

prediction = translate(text.lower())
print(prediction)