hanneshapke

import base64
import googleapiclient.discovery

from example_pb2 import Example
from feature_pb2 import BytesList, Feature, Features


def _convert_to_pb(value):
    """ Serialize a given sentence to the ProtoBuf Structure required to model the tf.Example data structure.
    Feel free to add more features and different data types if your models reqiures different inputs. An overview of 

hanneshapke

export CURRENT_DEV=kreuzberg
alias latest_dev='cd ~/development/$CURRENT_DEV'

# ssh tunnel
alias ssd='~/bin/ssh_host_color.sh ubuntu@remote -p 823 -L 6006:gpu:6006'

# add additional paths to the PYTHONPATH
export PYTHONPATH=$PYTHONPATH:~/development/additional_package

# git shortcuts

hanneshapke

import bz2
import pickle

from django.conf import settings
from djang_redis import get_redis_connection
from tqdm import tqdm

from .constants import GOOGLE_WORD2VEC_MODEL_NAME

hanneshapke

import bz2
import numpy as np
import pickle

from django.conf import settings
from django_redis import get_redis_connection
from gensim.models.keyedvectors import KeyedVectors

from .constants import GOOGLE_WORD2VEC_MODEL_NAME
from .redis import load_word2vec_model_into_redis, query_redis

hanneshapke

def get_highlighted_tokens(tokens, matrix, model, layer_name, threshold, y_labels):
    indices = get_token_indices(model, layer_name, threshold, matrix, y_labels)
    
    ctokens = []
    for i, t in enumerate(tokens):
        if i in indices.keys():
            _color = color(indices[i], threshold=threshold)
            ctokens.append(cstr(t, color=_color))
        else:
            ctokens.append(t)

hanneshapke

def get_token_indices(model, layer_name, threshold, matrix, y_labels):
    heatmap = get_heatmap(model=model, layer_name=layer_name, matrix=matrix, y_labels=y_labels)
    _, output_dim = get_conv_layer(model, layer_name)

    # depending on the ration between the input and layer output shape, we need to calculate 
    # how many original tokens have contributed to the layer output
    dim_ratio = matrix.shape[1] / output_dim
    if dim_ratio < 1.5:
        window_size = 1
    else:

hanneshapke

def color(hvalue, threshold, max=1, cdefault='black', colors=['red', 'yellow', 'green', 'cyan', 'blue']):
    num_colors = len(colors)
    if hvalue < threshold:
        return cdefault
    for i, color in enumerate(colors):
        if hvalue > (max - (max - threshold) / num_colors * (i + 1)):
            return color

hanneshapke

def cstr(s, color='black'):
    return "<text style=\"color:{}\">{}</text>".format(color, s)

hanneshapke

def plot_heatmap(heatmap, height_ratio=0.05):
        # calculating how often the vector should be repeated to display a height relative to the vector length
        repeat_vector_n_times = int(heatmap.shape[0] * height_ratio)
        plt.matshow([heatmap] * repeat_vector_n_times)

hanneshapke

def norm_heatmap(heatmap): 
    # element-wise maximum calculation, basically setting all negative values to zero
    heatmap = np.maximum(heatmap, 0)
    # normalizing the heatmap to values between 0 and 1
    norm_heatmap = heatmap / np.max(heatmap)
    return norm_heatmap