hiddenOutput cudnn LSTM

README.md

Simple test to figure out the format of hiddenOutput of cudnn LSTM, see the file below.

hiddenOutput of normal LSTM

(1,.,.) =
  0.3042  0.3042  0.3042
[torch.CudaTensor of size 1x1x3]

hiddenOutput of bidirection LSTM

(1,.,.) =
  0.3042  0.3042  0.3042

(2,.,.) =
  0.2694  0.2694  0.2694
[torch.CudaTensor of size 2x1x3]

hiddenOutput of two layer bidirection LSTM

(1,.,.) =
  0.3042  0.3042  0.3042

(2,.,.) =
  0.2694  0.2694  0.2694

(3,.,.) =
  0.2341  0.2341  0.2341

(4,.,.) =
  0.2274  0.2274  0.2274
[torch.CudaTensor of size 4x1x3]

example.lua

-- cudnn Bi-directional hiddenOutput order
-- This file demonstrates the hiddenOutput order of BLSTM
-- the output of BLSTM is 2*LxBxH
-- Where the order of the first dimension is as follows:
-- first layer is the activation of the first normal LSTM layer
-- the second layer is the activation of the first reversed LSTM layer
-- third layer is the activation of the second normal LSTM layer
-- the fourth layer is the activation of the second reversed LSTM layer
--
-- The below file demonstrates this
require 'nn'
require 'cutorch'
require 'cunn'
require 'cudnn'

local model1 = cudnn.LSTM(3, 3, 1)
local model2 = cudnn.BLSTM(3, 3, 1)
local model3 = cudnn.BLSTM(3, 3, 2)
local params1, _ = model1:getParameters()
local params2, _ = model2:getParameters()
local params3, _ = model3:getParameters()
params1:fill(0.1)
params2:fill(0.1)
params3:fill(0.1)
local input = torch.Tensor{{1, 2, 3}}:t()
local lookup = nn.LookupTable(3, 3)
local input2 = lookup:forward(input):cuda()
model1:forward(input2)
print("hiddenOutput of normal LSTM")
print(model1.hiddenOutput)
model2:forward(input2)
print("hiddenOutput of bidirection LSTM")
print(model2.hiddenOutput)
print("hiddenOutput of two layer bidirection LSTM")
model3:forward(input2)
print(model3.hiddenOutput)