tensorflow_nn_auto_ml_prototype.py

from tensorflow.keras import Model
import tensorflow as tf
import numpy as np
import pandas as pd
import random

class ReluDense(tf.Module):
    def __init__(self, in_features, out_features, name=None):
        super().__init__(name=name)
        self.w = tf.Variable(
            tf.random.normal([out_features, out_features]), name='w'
        )
        self.b = tf.Variable(
            tf.zeros([out_features]), name='b'
        )
    
    def __call__(self, x):
        y = tf.matmul(x, self.w) + self.b
        return tf.nn.relu(y)

class Dense(tf.Module):
    def __init__(self, in_features, out_features, name=None):
        super().__init__(name=name)
        self.w = tf.Variable(
            tf.random.normal([out_features, out_features]), name='w'
        )
        self.b = tf.Variable(
            tf.zeros([out_features]), name='b'
        )
    
    def __call__(self, x):
        return tf.matmul(x, self.w) + self.b
    
class NeuralNet(Model):
    def __init__(self, X_in, X_out, optimizer, dropout=0.1):
        super(NeuralNet, self).__init__()
        self.number_of_relu_dense = 0
        self.number_of_vanilla_dense = 0
        self.relu_layers = []
        self.dense_layers = []
        for _ in range(1, random.randint(2, 10)):
            self.relu_layers.append(ReluDense(X_in, X_out))
            self.number_of_relu_dense += 1
        for _ in range(1, random.randint(2, 10)):
            self.dense_layers.append(Dense(X_in, X_out))
            self.number_of_vanilla_dense += 1
        self.out = Dense(X_in, X_out)
        self.optimizer = optimizer
        self.dropout = dropout

    def call(self, x, train=False):
        if train:
            for layer in self.relu_layers:
                x = layer(x)
                x = dropout(x, self.dropout)
            for layer in self.dense_layers:
                x = layer(x)
                x = dropout(x, self.dropout)
        else:
            for layer in self.relu_layers:
                x = layer(x)
                x = dropout(x, self.dropout)
            for layer in self.dense_layers:
                x = layer(x)
                x = dropout(x, self.dropout)
            x = self.out(x)
        return tf.reduce_mean(x, axis=1)

    def step(self, x, y):
        x = tf.cast(x, tf.float32)
        y = tf.cast(y, tf.float32)
        with tf.GradientTape() as tape:
            pred = self.call(x)
            loss = mse(pred, y)

        gradients = tape.gradient(loss, self.trainable_variables)
        self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))

def dropout(X, drop_probability):
    keep_probability = 1 - drop_probability
    mask = np.random.uniform(0, 1.0) < keep_probability
    if keep_probability > 0.0:
        scale = (1/keep_probability)
    else:
        scale = 0.0
    return mask * X * scale

def mse(x, y):
    x = tf.cast(x, tf.float64)
    y = tf.cast(y, tf.float64)
    return tf.metrics.MSE(x, y)

def maape(y, y_pred):
    normed_absolute_error = np.abs(y_pred - y) / y
    normed_arctan_abs_error = np.arctan(normed_absolute_error)
    return np.sum(normed_arctan_abs_error)


if __name__ == '__main__':
    
    X = pd.read_csv("X.csv")
    y = np.load("y.npy")
    X = X.values
    X_val = pd.read_csv("X_val.csv")
    X_val = X_val.values
    y_val = np.load("y_val.npy")
    mse_loss = []
    maape_loss = []
    stopped = []
    optimizer = tf.optimizers.Adam(0.9)
    neural_nets = [NeuralNet(X.shape[0], X.shape[1], optimizer)
                   for _ in range(50)]

    for step in range(300):
        for index in range(len(neural_nets)):
            if index in stopped:
                continue
            nn = neural_nets[index]
            nn.step(X, y)
            pred = nn(X_val)
            loss_mse = mse(pred, y_val)
            loss_maape = maape(pred, y_val)
            is_nan = pd.isnull(loss_mse.numpy()) or pd.isnull(loss_maape)
            is_less_than_zero = loss_mse.numpy() < 0 or loss_maape < 0
            if is_nan or is_less_than_zero:
                stopped.append(index)
                continue
            mse_loss.append(loss_mse)
            maape_loss.append(loss_maape)

    min_index = mse_loss.index(min(mse_loss))
    print(mse_loss[min_index].numpy())
    print(maape_loss[min_index])
    print(len(stopped))