dense.py

dense.py

class Dense:
    def __init__(self, units, input_dim, kernel_initializer='he_normal', bias_initializer='zeros'):
        self.units = units
        self.input_dim = input_dim
        self.kernel_initializer = kernel_initializer
        self.bias_initializer = bias_initializer
        self.W = None
        self.b = None
        self.dW = None
        self.db = None
        self.initialize_weights()
        
        
    def sigma(self, initializer, fan_in, fan_out):
        """重みと閾値の初期化用にガウス分布の標準偏差値を返す
           参照: https://keras.io/ja/initializers
        """
        if initializer=="glorot_normal": # for sigmoid, tanh.
            return np.sqrt(2. / (fan_in + fan_out))
        elif initializer=="he_normal":  # for relu
            return np.sqrt(2. / fan_in)
        elif initializer=="lecun_normal":  # for traial
            return np.sqrt(1. / fan_in)
        elif initializer=="one_normal":  # for traial
            return 1.0
        elif initializer=="zeros":
            return 0.0
        
        
    def initialize_weights(self):
        """重み・閾値の初期化"""
        self.W = np.random.randn(self.input_dim, self.units)*self.sigma(self.kernel_initializer, self.input_dim, self.units)
        self.b = np.random.randn(1, self.units)*self.sigma(self.bias_initializer, self.input_dim, self.units)
    
    
    def forward_prop(self, Phi):
        """順伝播演算"""
        Z = np.dot(Phi, self.W) + self.b
        return Z
    
    
    def back_prop(self, Phi, Delta):
        """逆伝播演算"""
        n_samples = Delta.shape[0]  # サンプル数で割って1サンプル当たりの平均値にする
        self.dW = np.dot(Phi.T, Delta)/n_samples
        self.db = np.dot(np.ones([1, n_samples]), Delta)/n_samples
        dPhi = np.dot(Delta, self.W.T)                                                   
        return dPhi
    
    
    def update_weights(self, mu):
        """重み・閾値アップデート"""
        self.W -= mu * self.dW
        self.b -= mu * self.db