Skip to content

Instantly share code, notes, and snippets.

@melgor

melgor/check_model_at_cpy.ipynb

Last active June 25, 2021 04:32
Show Gist options
  • Save melgor/41e7d9367410b71dfddc33db34cba85f to your computer and use it in GitHub Desktop.
Save melgor/41e7d9367410b71dfddc33db34cba85f to your computer and use it in GitHub Desktop.
Download ZIP
How to restore CUDNNLSTM of TensorFlow at CPU device? So that it could be used in GPU, CPU or Mobile
Display the source blob
Display the rendered blob
Raw
check_model_at_cpy.ipynb
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Display the source blob
Display the rendered blob
Raw
restore_cudnn.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import tensorflow as tf\n",
"import numpy as np\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"num_layers = 1\n",
"num_units = 128\n",
"direction = \"bidirectional\""
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"inputs\n",
"transpose/perm\n",
"transpose\n",
"cudnn_lstm/cudnn_lstm/random_uniform/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_1/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_1/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_1/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_1/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_1/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_1/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_1\n",
"cudnn_lstm/cudnn_lstm/random_uniform_2/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_2/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_2/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_2/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_2/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_2/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_2\n",
"cudnn_lstm/cudnn_lstm/random_uniform_3/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_3/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_3/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_3/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_3/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_3/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_3\n",
"cudnn_lstm/cudnn_lstm/random_uniform_4/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_4/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_4/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_4/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_4/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_4/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_4\n",
"cudnn_lstm/cudnn_lstm/random_uniform_5/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_5/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_5/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_5/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_5/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_5/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_5\n",
"cudnn_lstm/cudnn_lstm/random_uniform_6/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_6/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_6/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_6/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_6/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_6/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_6\n",
"cudnn_lstm/cudnn_lstm/random_uniform_7/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_7/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_7/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_7/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_7/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_7/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_7\n",
"cudnn_lstm/cudnn_lstm/random_uniform_8/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_8/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_8/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_8/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_8/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_8/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_8\n",
"cudnn_lstm/cudnn_lstm/random_uniform_9/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_9/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_9/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_9/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_9/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_9/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_9\n",
"cudnn_lstm/cudnn_lstm/random_uniform_10/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_10/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_10/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_10/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_10/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_10/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_10\n",
"cudnn_lstm/cudnn_lstm/random_uniform_11/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_11/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_11/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_11/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_11/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_11/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_11\n",
"cudnn_lstm/cudnn_lstm/random_uniform_12/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_12/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_12/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_12/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_12/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_12/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_12\n",
"cudnn_lstm/cudnn_lstm/random_uniform_13/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_13/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_13/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_13/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_13/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_13/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_13\n",
"cudnn_lstm/cudnn_lstm/random_uniform_14/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_14/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_14/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_14/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_14/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_14/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_14\n",
"cudnn_lstm/cudnn_lstm/random_uniform_15/shape\n",
"cudnn_lstm/cudnn_lstm/random_uniform_15/min\n",
"cudnn_lstm/cudnn_lstm/random_uniform_15/max\n",
"cudnn_lstm/cudnn_lstm/random_uniform_15/RandomUniform\n",
"cudnn_lstm/cudnn_lstm/random_uniform_15/sub\n",
"cudnn_lstm/cudnn_lstm/random_uniform_15/mul\n",
"cudnn_lstm/cudnn_lstm/random_uniform_15\n",
"cudnn_lstm/cudnn_lstm/Const\n",
"cudnn_lstm/cudnn_lstm/Const_1\n",
"cudnn_lstm/cudnn_lstm/Const_2\n",
"cudnn_lstm/cudnn_lstm/Const_3\n",
"cudnn_lstm/cudnn_lstm/Const_4\n",
"cudnn_lstm/cudnn_lstm/Const_5\n",
"cudnn_lstm/cudnn_lstm/Const_6\n",
"cudnn_lstm/cudnn_lstm/Const_7\n",
"cudnn_lstm/cudnn_lstm/Const_8\n",
"cudnn_lstm/cudnn_lstm/Const_9\n",
"cudnn_lstm/cudnn_lstm/Const_10\n",
"cudnn_lstm/cudnn_lstm/Const_11\n",
"cudnn_lstm/cudnn_lstm/Const_12\n",
"cudnn_lstm/cudnn_lstm/Const_13\n",
"cudnn_lstm/cudnn_lstm/Const_14\n",
"cudnn_lstm/cudnn_lstm/Const_15\n",
"cudnn_lstm/cudnn_lstm/CudnnRNNCanonicalToParams/num_layers\n",
"cudnn_lstm/cudnn_lstm/CudnnRNNCanonicalToParams/num_units\n",
"cudnn_lstm/cudnn_lstm/CudnnRNNCanonicalToParams/input_size\n",
"cudnn_lstm/cudnn_lstm/CudnnRNNCanonicalToParams\n",
"cudnn_lstm/opaque_kernel\n",
"cudnn_lstm/opaque_kernel/Assign\n",
"cudnn_lstm/opaque_kernel/read\n",
"cudnn_lstm/CudnnRNNParamsToCanonical/num_layers\n",
"cudnn_lstm/CudnnRNNParamsToCanonical/num_units\n",
"cudnn_lstm/CudnnRNNParamsToCanonical/input_size\n",
"cudnn_lstm/CudnnRNNParamsToCanonical\n",
"cudnn_lstm/concat/axis\n",
"cudnn_lstm/concat\n",
"cudnn_lstm/concat_1/axis\n",
"cudnn_lstm/concat_1\n",
"cudnn_lstm/concat_2/axis\n",
"cudnn_lstm/concat_2\n",
"cudnn_lstm/concat_3/axis\n",
"cudnn_lstm/concat_3\n",
"cudnn_lstm/concat_4/axis\n",
"cudnn_lstm/concat_4\n",
"cudnn_lstm/transpose/Rank\n",
"cudnn_lstm/transpose/sub/y\n",
"cudnn_lstm/transpose/sub\n",
"cudnn_lstm/transpose/Range/start\n",
"cudnn_lstm/transpose/Range/delta\n",
"cudnn_lstm/transpose/Range\n",
"cudnn_lstm/transpose/sub_1\n",
"cudnn_lstm/transpose\n",
"cudnn_lstm/add\n",
"cudnn_lstm/add_1\n",
"cudnn_lstm/add_2\n",
"cudnn_lstm/add_3\n",
"cudnn_lstm/concat_5/axis\n",
"cudnn_lstm/concat_5\n",
"cudnn_lstm/concat_6/axis\n",
"cudnn_lstm/concat_6\n",
"cudnn_lstm/concat_7/axis\n",
"cudnn_lstm/concat_7\n",
"cudnn_lstm/concat_8/axis\n",
"cudnn_lstm/concat_8\n",
"cudnn_lstm/concat_9/axis\n",
"cudnn_lstm/concat_9\n",
"cudnn_lstm/concat_10/axis\n",
"cudnn_lstm/concat_10\n",
"cudnn_lstm/transpose_1/Rank\n",
"cudnn_lstm/transpose_1/sub/y\n",
"cudnn_lstm/transpose_1/sub\n",
"cudnn_lstm/transpose_1/Range/start\n",
"cudnn_lstm/transpose_1/Range/delta\n",
"cudnn_lstm/transpose_1/Range\n",
"cudnn_lstm/transpose_1/sub_1\n",
"cudnn_lstm/transpose_1\n",
"cudnn_lstm/add_4\n",
"cudnn_lstm/add_5\n",
"cudnn_lstm/add_6\n",
"cudnn_lstm/add_7\n",
"cudnn_lstm/concat_11/axis\n",
"cudnn_lstm/concat_11\n",
"cudnn_lstm/Identity\n",
"cudnn_lstm/Shape\n",
"cudnn_lstm/strided_slice/stack\n",
"cudnn_lstm/strided_slice/stack_1\n",
"cudnn_lstm/strided_slice/stack_2\n",
"cudnn_lstm/strided_slice\n",
"cudnn_lstm/zeros/mul/x\n",
"cudnn_lstm/zeros/mul\n",
"cudnn_lstm/zeros/mul_1/y\n",
"cudnn_lstm/zeros/mul_1\n",
"cudnn_lstm/zeros/Less/y\n",
"cudnn_lstm/zeros/Less\n",
"cudnn_lstm/zeros/packed/0\n",
"cudnn_lstm/zeros/packed/2\n",
"cudnn_lstm/zeros/packed\n",
"cudnn_lstm/zeros/Const\n",
"cudnn_lstm/zeros\n",
"cudnn_lstm/zeros_1/mul/x\n",
"cudnn_lstm/zeros_1/mul\n",
"cudnn_lstm/zeros_1/mul_1/y\n",
"cudnn_lstm/zeros_1/mul_1\n",
"cudnn_lstm/zeros_1/Less/y\n",
"cudnn_lstm/zeros_1/Less\n",
"cudnn_lstm/zeros_1/packed/0\n",
"cudnn_lstm/zeros_1/packed/2\n",
"cudnn_lstm/zeros_1/packed\n",
"cudnn_lstm/zeros_1/Const\n",
"cudnn_lstm/zeros_1\n",
"cudnn_lstm/CudnnRNN\n",
"transpose_1/perm\n",
"transpose_1\n",
"gradients/Shape\n",
"gradients/grad_ys_0\n",
"gradients/Fill\n",
"gradients/transpose_1_grad/InvertPermutation\n",
"gradients/transpose_1_grad/transpose\n",
"gradients/zeros_like\n",
"gradients/zeros_like_1\n",
"gradients/zeros_like_2\n",
"gradients/cudnn_lstm/CudnnRNN_grad/CudnnRNNBackprop\n",
"gradients/cudnn_lstm/CudnnRNN_grad/tuple/group_deps\n",
"gradients/cudnn_lstm/CudnnRNN_grad/tuple/control_dependency\n",
"gradients/cudnn_lstm/CudnnRNN_grad/tuple/control_dependency_1\n",
"gradients/cudnn_lstm/CudnnRNN_grad/tuple/control_dependency_2\n",
"gradients/cudnn_lstm/CudnnRNN_grad/tuple/control_dependency_3\n",
"beta1_power/initial_value\n",
"beta1_power\n",
"beta1_power/Assign\n",
"beta1_power/read\n",
"beta2_power/initial_value\n",
"beta2_power\n",
"beta2_power/Assign\n",
"beta2_power/read\n",
"IsVariableInitialized\n",
"cond/Switch\n",
"cond/switch_t\n",
"cond/switch_f\n",
"cond/pred_id\n",
"cond/read\n",
"cond/read/Switch\n",
"cond/Switch_1\n",
"cond/Merge\n",
"Shape\n",
"zeros/Const\n",
"zeros\n",
"cudnn_lstm/opaque_kernel/Adam\n",
"cudnn_lstm/opaque_kernel/Adam/IsVariableInitialized\n",
"cudnn_lstm/opaque_kernel/Adam/cond/Switch\n",
"cudnn_lstm/opaque_kernel/Adam/cond/switch_t\n",
"cudnn_lstm/opaque_kernel/Adam/cond/switch_f\n",
"cudnn_lstm/opaque_kernel/Adam/cond/pred_id\n",
"cudnn_lstm/opaque_kernel/Adam/cond/read\n",
"cudnn_lstm/opaque_kernel/Adam/cond/read/Switch\n",
"cudnn_lstm/opaque_kernel/Adam/cond/Switch_1\n",
"cudnn_lstm/opaque_kernel/Adam/cond/Merge\n",
"cudnn_lstm/opaque_kernel/Adam/cond/read/Switch_cudnn_lstm/opaque_kernel/Adam_0\n",
"cudnn_lstm/opaque_kernel/Adam/cond/read_cudnn_lstm/opaque_kernel/Adam_0\n",
"cudnn_lstm/opaque_kernel/Adam/cond/Merge_cudnn_lstm/opaque_kernel/Adam_0\n",
"cudnn_lstm/opaque_kernel/Adam/Shape_cudnn_lstm/opaque_kernel/Adam_0\n",
"cudnn_lstm/opaque_kernel/Adam/zeros_cudnn_lstm/opaque_kernel/Adam_0\n",
"cudnn_lstm/opaque_kernel/Adam/Assign\n",
"cudnn_lstm/opaque_kernel/Adam/read\n",
"IsVariableInitialized_1\n",
"cond_1/Switch\n",
"cond_1/switch_t\n",
"cond_1/switch_f\n",
"cond_1/pred_id\n",
"cond_1/read\n",
"cond_1/read/Switch\n",
"cond_1/Switch_1\n",
"cond_1/Merge\n",
"Shape_1\n",
"zeros_1/Const\n",
"zeros_1\n",
"cudnn_lstm/opaque_kernel/Adam_1\n",
"cudnn_lstm/opaque_kernel/Adam_1/IsVariableInitialized\n",
"cudnn_lstm/opaque_kernel/Adam_1/cond/Switch\n",
"cudnn_lstm/opaque_kernel/Adam_1/cond/switch_t\n",
"cudnn_lstm/opaque_kernel/Adam_1/cond/switch_f\n",
"cudnn_lstm/opaque_kernel/Adam_1/cond/pred_id\n",
"cudnn_lstm/opaque_kernel/Adam_1/cond/read\n",
"cudnn_lstm/opaque_kernel/Adam_1/cond/read/Switch\n",
"cudnn_lstm/opaque_kernel/Adam_1/cond/Switch_1\n",
"cudnn_lstm/opaque_kernel/Adam_1/cond/Merge\n",
"cudnn_lstm/opaque_kernel/Adam_1/cond_1/read/Switch_cudnn_lstm/opaque_kernel/Adam_1_0\n",
"cudnn_lstm/opaque_kernel/Adam_1/cond_1/read_cudnn_lstm/opaque_kernel/Adam_1_0\n",
"cudnn_lstm/opaque_kernel/Adam_1/cond_1/Merge_cudnn_lstm/opaque_kernel/Adam_1_0\n",
"cudnn_lstm/opaque_kernel/Adam_1/Shape_1_cudnn_lstm/opaque_kernel/Adam_1_0\n",
"cudnn_lstm/opaque_kernel/Adam_1/zeros_1_cudnn_lstm/opaque_kernel/Adam_1_0\n",
"cudnn_lstm/opaque_kernel/Adam_1/Assign\n",
"cudnn_lstm/opaque_kernel/Adam_1/read\n",
"Adam/learning_rate\n",
"Adam/beta1\n",
"Adam/beta2\n",
"Adam/epsilon\n",
"Adam/update_cudnn_lstm/opaque_kernel/ApplyAdam\n",
"Adam/mul\n",
"Adam/Assign\n",
"Adam/mul_1\n",
"Adam/Assign_1\n",
"Adam\n",
"save/Const\n",
"save/SaveV2/tensor_names\n",
"save/SaveV2/shape_and_slices\n",
"save/SaveV2\n",
"save/control_dependency\n",
"save/RestoreV2/tensor_names\n",
"save/RestoreV2/shape_and_slices\n",
"save/RestoreV2\n",
"save/Assign\n",
"save/Assign_1\n",
"save/Assign_2\n",
"save/Assign_3\n",
"save/Assign_4\n",
"save/transpose/Rank\n",
"save/transpose/sub/y\n",
"save/transpose/sub\n",
"save/transpose/Range/start\n",
"save/transpose/Range/delta\n",
"save/transpose/Range\n",
"save/transpose/sub_1\n",
"save/transpose\n",
"save/Const_1\n",
"save/split/split_dim\n",
"save/split\n",
"save/Const_2\n",
"save/split_1/split_dim\n",
"save/split_1\n",
"save/Const_3\n",
"save/split_2/split_dim\n",
"save/split_2\n",
"save/Const_4\n",
"save/split_3/split_dim\n",
"save/split_3\n",
"save/Const_5\n",
"save/split_4/split_dim\n",
"save/split_4\n",
"save/Const_6\n",
"save/split_5/split_dim\n",
"save/split_5\n",
"save/mul/y\n",
"save/mul\n",
"save/mul_1/y\n",
"save/mul_1\n",
"save/mul_2/y\n",
"save/mul_2\n",
"save/mul_3/y\n",
"save/mul_3\n",
"save/transpose_1/Rank\n",
"save/transpose_1/sub/y\n",
"save/transpose_1/sub\n",
"save/transpose_1/Range/start\n",
"save/transpose_1/Range/delta\n",
"save/transpose_1/Range\n",
"save/transpose_1/sub_1\n",
"save/transpose_1\n",
"save/Const_7\n",
"save/split_6/split_dim\n",
"save/split_6\n",
"save/Const_8\n",
"save/split_7/split_dim\n",
"save/split_7\n",
"save/Const_9\n",
"save/split_8/split_dim\n",
"save/split_8\n",
"save/Const_10\n",
"save/split_9/split_dim\n",
"save/split_9\n",
"save/Const_11\n",
"save/split_10/split_dim\n",
"save/split_10\n",
"save/Const_12\n",
"save/split_11/split_dim\n",
"save/split_11\n",
"save/mul_4/y\n",
"save/mul_4\n",
"save/mul_5/y\n",
"save/mul_5\n",
"save/mul_6/y\n",
"save/mul_6\n",
"save/mul_7/y\n",
"save/mul_7\n",
"save/Reshape/shape\n",
"save/Reshape\n",
"save/Reshape_1/shape\n",
"save/Reshape_1\n",
"save/Reshape_2/shape\n",
"save/Reshape_2\n",
"save/Reshape_3/shape\n",
"save/Reshape_3\n",
"save/Reshape_4/shape\n",
"save/Reshape_4\n",
"save/Reshape_5/shape\n",
"save/Reshape_5\n",
"save/Reshape_6/shape\n",
"save/Reshape_6\n",
"save/Reshape_7/shape\n",
"save/Reshape_7\n",
"save/Reshape_8/shape\n",
"save/Reshape_8\n",
"save/Reshape_9/shape\n",
"save/Reshape_9\n",
"save/Reshape_10/shape\n",
"save/Reshape_10\n",
"save/Reshape_11/shape\n",
"save/Reshape_11\n",
"save/Reshape_12/shape\n",
"save/Reshape_12\n",
"save/Reshape_13/shape\n",
"save/Reshape_13\n",
"save/Reshape_14/shape\n",
"save/Reshape_14\n",
"save/Reshape_15/shape\n",
"save/Reshape_15\n",
"save/CudnnRNNCanonicalToParams/num_layers\n",
"save/CudnnRNNCanonicalToParams/num_units\n",
"save/CudnnRNNCanonicalToParams/input_size\n",
"save/CudnnRNNCanonicalToParams\n",
"save/Assign_5\n",
"save/restore_all\n",
"init\n"
]
}
],
"source": [
"inputs = tf.placeholder(tf.float32, [None, None, 32], name=\"inputs\")\n",
"convolved = tf.transpose(inputs, [1, 0, 2])\n",
"lstm = tf.contrib.cudnn_rnn.CudnnLSTM(num_layers, num_units, direction=direction)\n",
"outputs, output_states = lstm(convolved, training=True)\n",
"enc_output = tf.transpose(outputs, [1, 0, 2])\n",
"optimizer = tf.train.AdamOptimizer(0.001)\n",
"optim = optimizer.minimize(enc_output)\n",
"\n",
"saver = tf.train.Saver()\n",
"init = tf.global_variables_initializer()\n",
"with tf.Session() as session:\n",
" session.run(init)\n",
" o = session.run([optim, enc_output], {inputs:np.zeros([10,100,32])})\n",
" saver.save(session, 'cudnn_cpu_bi/model_1')\n",
" # We can verify that we can access the list of operations in the graph\n",
" for op in session.graph.get_operations():\n",
" print(op.name)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tensor_name: beta1_power\n",
"0.80999994\n",
"tensor_name: beta2_power\n",
"0.99800104\n",
"tensor_name: cudnn_lstm/opaque_kernel\n",
"[-0.13538651 -0.13777976 0.00415914 ... 0. 0.\n",
" 0. ]\n",
"tensor_name: cudnn_lstm/opaque_kernel/Adam\n",
"[0. 0. 0. ... 0. 0. 0.]\n",
"tensor_name: cudnn_lstm/opaque_kernel/Adam_1\n",
"[0. 0. 0. ... 0. 0. 0.]\n",
"tensor_name: cudnn_lstm/stack_bidirectional_rnn/cell_0/bidirectional_rnn/bw/cudnn_compatible_lstm_cell/bias\n",
"[ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" 0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 0.002 -0.002\n",
" 0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. ]\n",
"tensor_name: cudnn_lstm/stack_bidirectional_rnn/cell_0/bidirectional_rnn/bw/cudnn_compatible_lstm_cell/kernel\n",
"[[-0.03987785 0.17737007 -0.17603733 ... -0.05294259 0.00885665\n",
" -0.00499246]\n",
" [-0.00834025 0.06144366 -0.04406044 ... 0.16670498 0.19086066\n",
" 0.15666988]\n",
" [ 0.08400673 0.14355147 -0.1895578 ... 0.10008124 0.07721415\n",
" -0.00239486]\n",
" ...\n",
" [-0.01313955 0.12693875 -0.08179953 ... -0.04146486 -0.01833126\n",
" 0.07861689]\n",
" [ 0.13157071 -0.13177846 0.06819822 ... -0.14118378 -0.0855393\n",
" -0.04774181]\n",
" [-0.0817693 -0.10887307 0.05002493 ... 0.02207133 0.07546543\n",
" -0.15090774]]\n",
"tensor_name: cudnn_lstm/stack_bidirectional_rnn/cell_0/bidirectional_rnn/fw/cudnn_compatible_lstm_cell/bias\n",
"[ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" 0.002 -0.002 -0.002 0.002 -0.002 0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 -0.002\n",
" -0.002 -0.002 -0.002 -0.002 -0.002 -0.002 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
" 0. 0. ]\n",
"tensor_name: cudnn_lstm/stack_bidirectional_rnn/cell_0/bidirectional_rnn/fw/cudnn_compatible_lstm_cell/kernel\n",
"[[-0.13538651 -0.08269153 -0.05373056 ... 0.01537631 0.02488364\n",
" -0.16640894]\n",
" [-0.13777976 0.09026167 0.14210978 ... -0.17184536 -0.13834968\n",
" -0.02148247]\n",
" [ 0.00415914 -0.04386127 -0.16173068 ... -0.00032383 0.0834586\n",
" -0.02253859]\n",
" ...\n",
" [-0.03001828 0.10460018 0.10160665 ... 0.05504832 0.03170046\n",
" 0.04891993]\n",
" [-0.00953992 0.09035453 0.04616997 ... 0.01980528 -0.06709683\n",
" 0.02004722]\n",
" [ 0.11823703 -0.03300903 -0.10799882 ... -0.1428191 0.06366883\n",
" 0.12410934]]\n"
]
}
],
"source": [
"from tensorflow.python.tools.inspect_checkpoint import print_tensors_in_checkpoint_file\n",
"# List ALL tensors.\n",
"print_tensors_in_checkpoint_file(file_name='cudnn_cpu_bi/model_1', tensor_name='', all_tensors=True)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"INFO:tensorflow:Restoring parameters from cudnn_cpu/model_1\n"
]
}
],
"source": [
"# Inference subgraph for unidirectional RNN on, e.g., CPU or mobile.\n",
"inputs = tf.placeholder(tf.float32, [None, None, 32], name=\"inputs\")\n",
"with tf.variable_scope(\"cudnn_lstm\"):\n",
" single_cell = lambda: tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell(num_units)\n",
" # NOTE: Even if there's only one layer, the cell needs to be wrapped in\n",
" # MultiRNNCell.\n",
" cell = tf.nn.rnn_cell.MultiRNNCell([single_cell() for _ in range(num_layers)])\n",
" # Leave the scope arg unset.\n",
" outputs, final_state = tf.nn.dynamic_rnn(cell, inputs,dtype=tf.float32)\n",
"saver = tf.train.Saver()\n",
"# Create session\n",
"sess = tf.Session()\n",
"# Restores\n",
"saver.restore(sess, 'cudnn_cpu/model_1')"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"INFO:tensorflow:Restoring parameters from cudnn_cpu_bi/model_1\n"
]
}
],
"source": [
"inputs = tf.placeholder(tf.float32, [None, None, 32], name=\"inputs\")\n",
"with tf.variable_scope(\"cudnn_lstm\"):\n",
" single_cell = lambda: tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell(num_units)\n",
" cells_fw = [single_cell() for _ in range(num_layers)]\n",
" cells_bw = [single_cell() for _ in range(num_layers)]\n",
" # Leave the scope arg unset.\n",
" (outputs, output_state_fw,\n",
" output_state_bw) = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, inputs,dtype=tf.float32)\n",
"\n",
" saver = tf.train.Saver()\n",
"# Create session\n",
"sess = tf.Session()\n",
"# Restores\n",
"saver.restore(sess, 'cudnn_cpu_bi/model_1')"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'cudnn_cpu_bi_trans/model_1'"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"saver.save(sess, 'cudnn_cpu_bi_trans/model_1')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
@melgor
Copy link
Author

melgor commented Jul 23, 2018

restore_cudnn.ipynb show how to save the model which use cudnn and then is restored using CudnnCompatibleLSTMCell (but not only GPU) cell.

@SysuJayce
Copy link

SysuJayce commented Oct 25, 2018 •
edited
Loading

Thank you for your tutorial!
However, after restoring cudnn_LSTM from checkpoint, I cant restore other variables from meta graph using something like this:

saver = tf.train.import_meta_graph("{}.meta".format(checkpoint))
saver.restore(sess, checkpoint)

That's to say, if I restore cudnn_LSTM in your way, then I cant restore other variables from meta graph.
And I need to get some variables like placeholder, accuracy for inference.

Can you tell me how to save and restore properly in this situation?
BTW, how can I restore cudnn variables to GPU directly? After googling a lot, I found that people just care about restoring cudnn variables to CPU, but I cant even restore it to GPU. Am I missing something? THANKS!

@sjtuldl
Copy link

sjtuldl commented Nov 7, 2018

Hi, I'm running into troubles when restore into CudnnCompatibleLSTMCell from CudnnLSTM, I've run the code above on my machine, and it raised an error, so I'm wandering if I'm using the version that do not support this feature. Would you please provide your tensorflow version that can run the code above? I'm currently run on Tensorflow 1.5.

@brennenhuang
Copy link

@SysuJayce I solved it, you should rebuild model and restore ckpt instead of using import_meta_graph

Copy link

ghost commented Jun 25, 2021

for anyone who had the same issue I did, remember to set time_major=True in stack_bidirectional_dynamic_rnn().

The CudnnLSTM() function is time major, as opposed to batch major. So remember to transpose your inputs and outputs (swap dimensions 0 and 1)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment