deepak-karkala

# Preprocessing pipeline for Numerical and Categorical features

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OrdinalEncoder
from sklearn.compose import ColumnTransformer


############ Pipeline for numerical features #############
# Numerical features

deepak-karkala

from sklearn.model_selection import learning_curve

# sklearn's learning_curve function can be used for following purposes
# 1. Learning curve: To study how training and validation error varies with more training examples
#      train_scores, valid_scores vs train_sizes
# 2. Model scalability: To study the time required to fit model as training data size increases
#      fit_times vs train_sizes
# 3. Model performance: To study how training error changes with time required to fit
#      train_scores vs fit_times

deepak-karkala

# Tune hyperparameters using Grid search and Randomised search Cross Validation

from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV

# Grid Search Cross Validation
# Specify discrete values for hyperparameters
param_grid = [
    {'max_depth': [1, 20, 100], 'max_features': [1, 5, 15, 20],

deepak-karkala

# Hyperparameter tuning using Tensorflow and Tensor board

# Testing performance with varioud optimizers
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'sgd', 'rmsprop']))
METRIC_ACCURACY = 'accuracy'
with tf.summary.create_file_writer('logs/hparam_tuning').as_default():
  hp.hparams_config(
    hparams=[HP_OPTIMIZER],
    metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],
  )

deepak-karkala

# Training dataset
train_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_masks))
train_dataset = train_dataset.map(load_image_train)
train_dataset = train_dataset.cache()
train_dataset = train_dataset.shuffle(BUFFER_SIZE)
train_dataset = train_dataset.batch(BATCH_SIZE)
train_dataset = train_dataset.repeat()
train_dataset = train_dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)

deepak-karkala

from tensorflow_examples.models.pix2pix import pix2pix

def build_model():
    LABEL_NAMES = np.asarray([
        'background', 'couch', 'chair', 'bed', 'vase', 'bowl', 'cup',
        'wine-glass', 'potted-plant'
    ])

    # Each pixel classified into one of OUTPUT_CHANNELS classes
    OUTPUT_CHANNELS = len(LABEL_NAMES) #3

deepak-karkala

### Tensorboard callbacks
# Log IoU at end of epoch
def log_epoch_metrics(epoch, logs):
  # Log directory
  logdir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
  file_writer = tf.summary.create_file_writer(logdir + "/fit")
  file_writer.set_as_default()

  # Intersection Over Union metric
  m = tf.keras.metrics.MeanIoU(num_classes=len(LABEL_NAMES))

deepak-karkala

# Model quantisation using TFLite
save_model_path = "/tmp/"

# Save original model in tflite format
tflite_models_dir = pathlib.Path(save_model_path)
tflite_models_dir.mkdir(exist_ok=True, parents=True)
converter = tf.lite.TFLiteConverter.from_keras_model(model)
tflite_model = converter.convert()
tflite_model_file = tflite_models_dir/"model.tflite"
print(tflite_model_file.write_bytes(tflite_model))

deepak-karkala

# Saving Tensorflow models for serving

# Option 1: SavedModel format
# Save model
model.save('saved_model/model')
# Load model
loaded_model = tf.keras.models.load_model('saved_model/model')

# Option 2: .h5 format
# Save model

deepak-karkala

# Serve Model predictions using REST API

# Predict and return JSON data
@app.route("/predict", methods=["POST"])
def predict():
    # initialize the data dictionary that will be returned from the view
    data = {"success": False}

    # ensure an image was properly uploaded to our endpoint
    if flask.request.method == "POST":