Character Embedding with 1D convolutions

char_embed.py

def apply_filter(x, filter_size, emb_dim, max_word_len):
    """
    Run convolution operation on x with "filter_size"d filter
    Args:
        x            : sequence to which conv should be applied 
        filter_size  : size of filter
        emb_dim      : embedding dimensions
        max_word_len : maximum length of a word
    Returns:
        Output of 1D convolution
    """
    batch_size_, seq_len_ = tf.shape(x)[0], tf.shape(x)[1]
    with tf.variable_scope('conv_{}'.format(filter_size)):
        # create filter based on filter_size
        filter_ = tf.get_variable('filter_x',
                         [filter_size, emb_dim, 1], tf.float32,
                         initializer=tf.contrib.layers.xavier_initializer()
                         )
        # perform 1D convolution
        #  return reshaped output
        return tf.reshape(
            tf.nn.conv1d(tf.reshape(x, [-1, max_word_len, emb_dim]),
                         filter_,
                         stride=1,
                         padding='VALID'),
            [batch_size_, seq_len_, -1])
      
def char_embed(char_sequence, char_vocab_size, emb_dim, max_word_len):
    """
    Character-level word representation using CNN
    Args:
        char_sequence   : sequence as list of characters
        char_vocab_size : size of character vocabulary
        emb_dim         : embedding dimension
        max_word_len    : maximum length of a word
    Returns:
        Character-level word embedding
    """
    # embedding matrix
    E = tf.get_variable('emb', shape = [char_vocab_size, emb_dim], 
            dtype=tf.float32, 
            initializer=tf.random_uniform_initializer(-0.01, 0.01))

    # embed sequence
    char_seq_emb = tf.nn.embedding_lookup(E, char_sequence)

    # list of filter sizes and count
    #filters = [ (3, 4), (2, 3), (4, 5) ]
    filters = [ (i, 25*i) for i in range(1, 7) ]
    convs = []
    # iterate through filters
    for filter_size, count in filters:
        convs_filter = []
        for i in range(count):
            with tf.variable_scope('iteration_{}'.format(i)):
                # apply filter, append to list of conv outputs
                convs_filter.append(
                        apply_filter(char_seq_emb, 
                            filter_size, emb_dim, max_word_len)
                        )

        # stack and append conv outputs
        convs.append(tf.reduce_sum(tf.stack(convs_filter, axis=2), axis=-1))

    # return concatenated conv outputs
    return tf.concat(convs, axis=-1)