justheuristic

"""
    "PyTorch must serve man, not rule over him" (c) DAO

A module for simple conversion between numpy and torch types.
Created out of frustration from lines like:
 - x = Variable(torch.FloatTensor(x)).cuda()                 # now var(x, 'float32')
 - (model(x).max(1)[1].data.cpu().numpy() == y).mean()       # now numpy(x)
"""

import torch

justheuristic

justheuristic

from models.transformer_fused import Model
from models.transformer_lm import TransformerLM
lm = TransformerLM('lm', out_voc, **{
        "hid_size": 256,
        "ff_size": 1024,
        "num_heads": 4,
        "num_layers": 4,
        "rescale_emb": True,
        "relu_dropout": 0.0,
        "res_dropout": 0.0,

justheuristic

justheuristic

"""
This is a TF implementation of constrained softmax from neural easy-first tagger, https://github.com/Unbabel/neural-easy-first
"""
import tensorflow as tf

def constrained_softmax(z, u, axis=-1, back_prop=True, swap_memory=False):
    """ 
    Computes softmax probs not exceeding constranints u
    Effectively it first computes normal softmax without constraints, 
    then enforces the constraints by 'cutting' probability mass that exceeds constraint, 

justheuristic

justheuristic

justheuristic

from minifox import MinFoxSolver
import matplotlib.pyplot as plt
%matplotlib inline
import numpy as np

# toy task: random projections
# first, we sample p-dimensional matrix M
# A is constructed as a random projection of M
# B is a random projection of first 4 components of M
# hence the "right answer" is to extract 5-th component of M from A

justheuristic

justheuristic