LSTMs suck

parity.py

import torch
from torch import nn
from torch.nn import functional as F
from torch.autograd import Variable
import sys

nlen = 5
model_type = nn.LSTM

running_loss = 1
epoch = 0
bs = 16

class Model(nn.Module):
    def __init__(self, rnn_maker):
        super(Model, self).__init__()
        self.nhid = 64
        self.l1 = nn.Linear(1, self.nhid)
        self.rnn = rnn_maker(self.nhid)
        self.l2 = nn.Linear(self.nhid, 1)

    def forward(self, inp):
        x = inp
        T = x.size(0)
        B = x.size(1)
        x = self.l1(x.view(T*B, 1)).view(T, B, self.nhid).tanh_()
        x, hids = self.rnn(x, None)
        x = self.l2(x[T-1].tanh()).sigmoid_()
        return x


model = Model(lambda x: model_type(x, x, num_layers = 1))
optim = torch.optim.Adam(model.parameters(), 1e-3)
while running_loss > 0.01:
    x = torch.rand(nlen, bs, 1).round()
    y = (x.sum(0) % 2).squeeze()

    x = Variable(x)
    y = Variable(y)

    output = model(x)
    loss = nn.functional.binary_cross_entropy(output, y)
    optim.zero_grad()
    loss.backward()
    optim.step()
    # print(x.size(), y.size(), output.size())
    # print(x.squeeze(), output.transpose(0, 1), y.unsqueeze(1).transpose(0, 1))

    running_loss = running_loss * 0.99 + loss.data[0] * 0.01
    if epoch % 100 == 0:
        print(epoch, running_loss)
    epoch += 1