decoder_layer.py

class DecoderLayer(Layer):
    def __init__(self, num_neurons, num_hidden_neurons, num_heads):
        super(DecoderLayer, self).__init__()

        # Build multi head attention layers and necessary additional layers
        self.multi_head_attention_layer1, self.attention_dropout1, self.attention_normalization1 =\
        build_multi_head_attention_layers(num_neurons, num_heads)   
        
        self.multi_head_attention_layer2, self.attention_dropout2, self.attention_normalization2 =\
        build_multi_head_attention_layers(num_neurons, num_heads)           

        # Build feed-forward neural network and necessary additional layers
        self.feed_forward_layer, self.feed_forward_dropout, self.feed_forward_normalization = \
        build_feed_forward_layers(num_neurons, num_hidden_neurons)

    def call(self, sequence, enconder_output, training, look_ahead_mask, padding_mask):

        attnention_output1, attnention_weights1 = self.multi_head_attention_layer1(sequence, sequence, sequence, look_ahead_mask)
        attnention_output1 = self.attention_dropout1(attnention_output1, training=training)
        attnention_output1 = self.attention_normalization1(sequence + attnention_output1)
        
        attnention_output2, attnention_weights2 = self.multi_head_attention_layer2(enconder_output, enconder_output, attnention_output1, padding_mask)
        attnention_output2 = self.attention_dropout1(attnention_output2, training=training)
        attnention_output2 = self.attention_normalization1(attnention_output1 + attnention_output2)

        output = self.feed_forward_layer(attnention_output2)
        output = self.feed_forward_dropout(output, training=training)
        output = self.feed_forward_normalization(attnention_output2 + output)

        return output, attnention_weights1, attnention_weights2