Skip to content

Instantly share code, notes, and snippets.

@tuxedocat

tuxedocat/GPU-memory-quota-KerasTF.ipynb

Created August 23, 2017 06:20
Show Gist options
  • Save tuxedocat/f3761974f9c1bcfd8af3f47d31d08ec9 to your computer and use it in GitHub Desktop.
Save tuxedocat/f3761974f9c1bcfd8af3f47d31d08ec9 to your computer and use it in GitHub Desktop.
Download ZIP
Minimal example for setting the limit of GPU memory usage when using Keras+TF
Display the source blob
Display the rendered blob
Raw
GPU-memory-quota-KerasTF.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using TensorFlow backend.\n"
]
}
],
"source": [
"import keras\n",
"import keras.backend as K\n",
"import tensorflow as tf\n",
"from time import sleep"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Keras = 2.0.6\n",
"TensorFlow = 1.2.1\n"
]
}
],
"source": [
"# Version\n",
"print('Keras = {}'.format(keras.__version__))\n",
"print('TensorFlow = {}'.format(tf.__version__))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Get configuration object with default params\n",
"config = tf.ConfigProto()\n",
"\n",
"# Maximum usage per process (quota)\n",
"config.gpu_options.per_process_gpu_memory_fraction = 0.25\n",
"\n",
"# Consume memory on demand (up to quota)\n",
"config.gpu_options.allow_growth = True\n",
"\n",
"# Set Keras's backend config\n",
"K.tensorflow_backend.set_session(tf.Session(config=config))"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wed Aug 23 15:17:25 2017 \r\n",
"+-----------------------------------------------------------------------------+\r\n",
"| NVIDIA-SMI 375.51 Driver Version: 375.51 |\r\n",
"|-------------------------------+----------------------+----------------------+\r\n",
"| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\r\n",
"| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\r\n",
"|===============================+======================+======================|\r\n",
"| 0 GeForce GTX 1080 Off | 0000:03:00.0 On | N/A |\r\n",
"| 0% 54C P2 45W / 240W | 176MiB / 8104MiB | 0% Default |\r\n",
"+-------------------------------+----------------------+----------------------+\r\n",
" \r\n",
"+-----------------------------------------------------------------------------+\r\n",
"| Processes: GPU Memory |\r\n",
"| GPU PID Type Process name Usage |\r\n",
"|=============================================================================|\r\n",
"| 0 2352 G /usr/lib/xorg/Xorg 59MiB |\r\n",
"| 0 2554 C /opt/miniconda3/envs/dl/bin/python 113MiB |\r\n",
"+-----------------------------------------------------------------------------+\r\n"
]
}
],
"source": [
"!nvidia-smi"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"model1 = keras.applications.VGG16()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wed Aug 23 15:17:37 2017 \r\n",
"+-----------------------------------------------------------------------------+\r\n",
"| NVIDIA-SMI 375.51 Driver Version: 375.51 |\r\n",
"|-------------------------------+----------------------+----------------------+\r\n",
"| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\r\n",
"| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\r\n",
"|===============================+======================+======================|\r\n",
"| 0 GeForce GTX 1080 Off | 0000:03:00.0 On | N/A |\r\n",
"| 0% 55C P2 46W / 240W | 2236MiB / 8104MiB | 0% Default |\r\n",
"+-------------------------------+----------------------+----------------------+\r\n",
" \r\n",
"+-----------------------------------------------------------------------------+\r\n",
"| Processes: GPU Memory |\r\n",
"| GPU PID Type Process name Usage |\r\n",
"|=============================================================================|\r\n",
"| 0 2352 G /usr/lib/xorg/Xorg 59MiB |\r\n",
"| 0 2554 C /opt/miniconda3/envs/dl/bin/python 2173MiB |\r\n",
"+-----------------------------------------------------------------------------+\r\n"
]
}
],
"source": [
"sleep(10)\n",
"!nvidia-smi"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true,
"scrolled": false
},
"outputs": [],
"source": [
"model2 = keras.applications.VGG16()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wed Aug 23 15:17:48 2017 \r\n",
"+-----------------------------------------------------------------------------+\r\n",
"| NVIDIA-SMI 375.51 Driver Version: 375.51 |\r\n",
"|-------------------------------+----------------------+----------------------+\r\n",
"| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\r\n",
"| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\r\n",
"|===============================+======================+======================|\r\n",
"| 0 GeForce GTX 1080 Off | 0000:03:00.0 On | N/A |\r\n",
"| 0% 56C P2 46W / 240W | 2236MiB / 8104MiB | 0% Default |\r\n",
"+-------------------------------+----------------------+----------------------+\r\n",
" \r\n",
"+-----------------------------------------------------------------------------+\r\n",
"| Processes: GPU Memory |\r\n",
"| GPU PID Type Process name Usage |\r\n",
"|=============================================================================|\r\n",
"| 0 2352 G /usr/lib/xorg/Xorg 59MiB |\r\n",
"| 0 2554 C /opt/miniconda3/envs/dl/bin/python 2173MiB |\r\n",
"+-----------------------------------------------------------------------------+\r\n"
]
}
],
"source": [
"sleep(10)\n",
"!nvidia-smi"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"scrolled": false
},
"outputs": [
{
"ename": "ResourceExhaustedError",
"evalue": "OOM when allocating tensor with shape[25088,4096]\n\t [[Node: fc1_2/kernel/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@fc1_2/kernel\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](fc1_2/kernel, fc1_2/random_uniform)]]\n\nCaused by op 'fc1_2/kernel/Assign', defined at:\n File \"/opt/miniconda3/envs/dl/lib/python3.5/runpy.py\", line 193, in _run_module_as_main\n \"__main__\", mod_spec)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/runpy.py\", line 85, in _run_code\n exec(code, run_globals)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/__main__.py\", line 3, in <module>\n app.launch_new_instance()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n app.start()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/kernelapp.py\", line 477, in start\n ioloop.IOLoop.instance().start()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n super(ZMQIOLoop, self).start()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tornado/ioloop.py\", line 888, in start\n handler_func(fd_obj, events)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n return fn(*args, **kwargs)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n self._handle_recv()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n self._run_callback(callback, msg)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n callback(*args, **kwargs)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n return fn(*args, **kwargs)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n return self.dispatch_shell(stream, msg)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n handler(stream, idents, msg)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n user_expressions, allow_stdin)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n res = shell.run_cell(code, store_history=store_history, silent=silent)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2698, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2802, in run_ast_nodes\n if self.run_code(code, result):\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2862, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"<ipython-input-9-c17359b0f90e>\", line 1, in <module>\n model3 = keras.applications.VGG16()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/applications/vgg16.py\", line 141, in VGG16\n x = Dense(4096, activation='relu', name='fc1')(x)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/engine/topology.py\", line 569, in __call__\n self.build(input_shapes[0])\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/layers/core.py\", line 825, in build\n constraint=self.kernel_constraint)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n return func(*args, **kwargs)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/engine/topology.py\", line 391, in add_weight\n weight = K.variable(initializer(shape), dtype=dtype, name=name)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\", line 321, in variable\n v = tf.Variable(value, dtype=_convert_string_dtype(dtype), name=name)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/ops/variables.py\", line 200, in __init__\n expected_shape=expected_shape)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/ops/variables.py\", line 309, in _init_from_args\n validate_shape=validate_shape).op\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/ops/state_ops.py\", line 271, in assign\n validate_shape=validate_shape)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/ops/gen_state_ops.py\", line 45, in assign\n use_locking=use_locking, name=name)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n op_def=op_def)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 2506, in create_op\n original_op=self._default_original_op, op_def=op_def)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 1269, in __init__\n self._traceback = _extract_stack()\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[25088,4096]\n\t [[Node: fc1_2/kernel/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@fc1_2/kernel\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](fc1_2/kernel, fc1_2/random_uniform)]]\n",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mResourceExhaustedError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1138\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1139\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1140\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1120\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1121\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1122\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/contextlib.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 67\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py\u001b[0m in \u001b[0;36mraise_exception_on_not_ok_status\u001b[0;34m()\u001b[0m\n\u001b[1;32m 465\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpywrap_tensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 466\u001b[0;31m pywrap_tensorflow.TF_GetCode(status))\n\u001b[0m\u001b[1;32m 467\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[25088,4096]\n\t [[Node: fc1_2/kernel/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@fc1_2/kernel\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](fc1_2/kernel, fc1_2/random_uniform)]]",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mResourceExhaustedError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-9-c17359b0f90e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapplications\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mVGG16\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/applications/vgg16.py\u001b[0m in \u001b[0;36mVGG16\u001b[0;34m(include_top, weights, input_tensor, input_shape, pooling, classes)\u001b[0m\n\u001b[1;32m 167\u001b[0m \u001b[0mWEIGHTS_PATH_NO_TOP\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 168\u001b[0m cache_subdir='models')\n\u001b[0;32m--> 169\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweights_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 170\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'theano'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[0mlayer_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert_all_kernels_in_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/engine/topology.py\u001b[0m in \u001b[0;36mload_weights\u001b[0;34m(self, filepath, by_name)\u001b[0m\n\u001b[1;32m 2570\u001b[0m \u001b[0mload_weights_from_hdf5_group_by_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2571\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2572\u001b[0;31m \u001b[0mload_weights_from_hdf5_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2573\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2574\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'close'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/engine/topology.py\u001b[0m in \u001b[0;36mload_weights_from_hdf5_group\u001b[0;34m(f, layers)\u001b[0m\n\u001b[1;32m 3046\u001b[0m ' elements.')\n\u001b[1;32m 3047\u001b[0m \u001b[0mweight_value_tuples\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msymbolic_weights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight_values\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3048\u001b[0;31m \u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_set_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight_value_tuples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3049\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3050\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36mbatch_set_value\u001b[0;34m(tuples)\u001b[0m\n\u001b[1;32m 2186\u001b[0m \u001b[0massign_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0massign_op\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2187\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0massign_placeholder\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2188\u001b[0;31m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0massign_ops\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36mget_session\u001b[0;34m()\u001b[0m\n\u001b[1;32m 166\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0m_MANUAL_VAR_INIT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 167\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 168\u001b[0;31m \u001b[0m_initialize_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 169\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_initialize_variables\u001b[0;34m()\u001b[0m\n\u001b[1;32m 339\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muninitialized_variables\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 340\u001b[0m \u001b[0msess\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 341\u001b[0;31m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariables_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muninitialized_variables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 787\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 788\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 789\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 790\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 791\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 995\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 996\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 997\u001b[0;31m feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m 998\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 999\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1130\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1131\u001b[0m return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1132\u001b[0;31m target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m 1133\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1134\u001b[0m return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
"\u001b[0;32m/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1150\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1151\u001b[0m \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1152\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1154\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[25088,4096]\n\t [[Node: fc1_2/kernel/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@fc1_2/kernel\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](fc1_2/kernel, fc1_2/random_uniform)]]\n\nCaused by op 'fc1_2/kernel/Assign', defined at:\n File \"/opt/miniconda3/envs/dl/lib/python3.5/runpy.py\", line 193, in _run_module_as_main\n \"__main__\", mod_spec)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/runpy.py\", line 85, in _run_code\n exec(code, run_globals)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/__main__.py\", line 3, in <module>\n app.launch_new_instance()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n app.start()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/kernelapp.py\", line 477, in start\n ioloop.IOLoop.instance().start()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n super(ZMQIOLoop, self).start()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tornado/ioloop.py\", line 888, in start\n handler_func(fd_obj, events)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n return fn(*args, **kwargs)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n self._handle_recv()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n self._run_callback(callback, msg)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n callback(*args, **kwargs)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n return fn(*args, **kwargs)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n return self.dispatch_shell(stream, msg)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n handler(stream, idents, msg)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n user_expressions, allow_stdin)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n res = shell.run_cell(code, store_history=store_history, silent=silent)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2698, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2802, in run_ast_nodes\n if self.run_code(code, result):\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2862, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"<ipython-input-9-c17359b0f90e>\", line 1, in <module>\n model3 = keras.applications.VGG16()\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/applications/vgg16.py\", line 141, in VGG16\n x = Dense(4096, activation='relu', name='fc1')(x)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/engine/topology.py\", line 569, in __call__\n self.build(input_shapes[0])\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/layers/core.py\", line 825, in build\n constraint=self.kernel_constraint)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n return func(*args, **kwargs)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/engine/topology.py\", line 391, in add_weight\n weight = K.variable(initializer(shape), dtype=dtype, name=name)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\", line 321, in variable\n v = tf.Variable(value, dtype=_convert_string_dtype(dtype), name=name)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/ops/variables.py\", line 200, in __init__\n expected_shape=expected_shape)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/ops/variables.py\", line 309, in _init_from_args\n validate_shape=validate_shape).op\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/ops/state_ops.py\", line 271, in assign\n validate_shape=validate_shape)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/ops/gen_state_ops.py\", line 45, in assign\n use_locking=use_locking, name=name)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n op_def=op_def)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 2506, in create_op\n original_op=self._default_original_op, op_def=op_def)\n File \"/opt/miniconda3/envs/dl/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 1269, in __init__\n self._traceback = _extract_stack()\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[25088,4096]\n\t [[Node: fc1_2/kernel/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@fc1_2/kernel\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](fc1_2/kernel, fc1_2/random_uniform)]]\n"
]
}
],
"source": [
"model3 = keras.applications.VGG16()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sleep(10)\n",
"!nvidia-smi"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python3.5 (DeepLearning-dev)",
"language": "python",
"name": "python3-dl"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment