LSTM in numpy http://chris-chris.ai/2017/10/10/LSTM-LayerNorm-breakdown-eng/

LSTM_in_numpy.py

import tensorflow as tf
import numpy as np
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import variables
from tensorflow.contrib.rnn.python.ops import rnn_cell

res = []

with tf.Session() as sess:
  with variable_scope.variable_scope(
      "other", initializer=init_ops.constant_initializer(0.5)) as vs:
    x = array_ops.zeros(
      [1, 3])  # Test BasicLSTMCell with input_size != num_units.
    c = array_ops.zeros([1, 2])
    h = array_ops.zeros([1, 2])
    state = (c, h)
    cell = rnn_cell.LayerNormBasicLSTMCell(2, layer_norm=False)
    g, out_m = cell(x, state)
    sess.run([variables.global_variables_initializer()])
    res = sess.run([g, out_m], {
      x.name: np.array([[1., 1., 1.]]),
      c.name: 0.1 * np.asarray([[0, 1]]),
      h.name: 0.1 * np.asarray([[2, 3]]),
    })

    print(res[1].c)
    print(res[1].h)

    expected_h = np.array([[ 0.64121795, 0.68166804]])
    expected_c = np.array([[ 0.88477188, 0.98103917]])

import numpy as np
x = np.array([[1., 1., 1.]])
c = 0.1 * np.asarray([[0, 1]])
h = 0.1 * np.asarray([[2, 3]])

num_units = 2

args = np.concatenate((x,h), axis=1)
print(args)

out_size = 4 * num_units
proj_size = args.shape[-1]
print(out_size)
print(proj_size)

weights = np.ones([proj_size, out_size]) * 0.5


print(weights)


out = np.matmul(args, weights)
print(out)

bias = np.ones([out_size]) * 0.5
print(bias)

concat = out + bias
print(concat)

i, j, f, o = np.split(concat, 4, 1)
print(i)
print(j)
print(f)
print(o)

g = np.tanh(j)
print(g)

def sigmoid_array(x):
  return 1 / (1 + np.exp(-x))

forget_bias = 1.0

sigmoid_f = sigmoid_array(f + forget_bias)
print(sigmoid_f)

sigmoid_array(i) * g

new_c = c * sigmoid_f + sigmoid_array(i) * g
print(new_c)

new_h = np.tanh(new_c) * sigmoid_array(o)
print(new_h)

print(new_h)
print(new_c)

print(res[1].h)
print(res[1].c)

np.testing.assert_almost_equal(res[1].h, np.array(new_h, dtype=np.float32))
np.testing.assert_almost_equal(res[1].c, np.array(new_c, dtype=np.float32))

Updated for Python 2 & 3 compatibility.