justheuristic

justheuristic

from minifox import MinFoxSolver
import matplotlib.pyplot as plt
%matplotlib inline
import numpy as np

# toy task: random projections
# first, we sample p-dimensional matrix M
# A is constructed as a random projection of M
# B is a random projection of first 4 components of M
# hence the "right answer" is to extract 5-th component of M from A

justheuristic

justheuristic

justheuristic

"""
This is a TF implementation of constrained softmax from neural easy-first tagger, https://github.com/Unbabel/neural-easy-first
"""
import tensorflow as tf

def constrained_softmax(z, u, axis=-1, back_prop=True, swap_memory=False):
    """ 
    Computes softmax probs not exceeding constranints u
    Effectively it first computes normal softmax without constraints, 
    then enforces the constraints by 'cutting' probability mass that exceeds constraint, 

justheuristic

justheuristic

from models.transformer_fused import Model
from models.transformer_lm import TransformerLM
lm = TransformerLM('lm', out_voc, **{
        "hid_size": 256,
        "ff_size": 1024,
        "num_heads": 4,
        "num_layers": 4,
        "rescale_emb": True,
        "relu_dropout": 0.0,
        "res_dropout": 0.0,

justheuristic

justheuristic

"""
    "PyTorch must serve man, not rule over him" (c) DAO

A module for simple conversion between numpy and torch types.
Created out of frustration from lines like:
 - x = Variable(torch.FloatTensor(x)).cuda()                 # now var(x, 'float32')
 - (model(x).max(1)[1].data.cpu().numpy() == y).mean()       # now numpy(x)
"""

import torch

justheuristic

@lru_cache()
def infer_batch_axes(model,
                     get_dummy_input=lambda bsize: {'inp': tf.ones([bsize, 3], dtype='int32'),
                                                    'inp_len': tf.constant([3]*bsize, dtype='int32')},
                     check_at=(3, 5, 7),
                     sess=None, **flags):
    """
    This function attempts to figure out batch dimensions by seeing what axes change on different batch sizes.
    It stands as a monument of hatred to a person who thought that time-major axes order in tensorflow RNN is cool.
    :param model: TranslateModel instance to figure out batch dimensions for.