Last active
July 25, 2018 14:32
-
-
Save RomanSteinberg/840d58b9333a3359cec43bbf4c464e5a to your computer and use it in GitHub Desktop.
Estimator fail because of folder
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import numpy as np | |
| import tensorflow as tf | |
| experiment_folder = '/output/' | |
| input_shape = [299, 299, 3] | |
| def imgs_input_fn(filenames, labels=None, perform_shuffle=False, repeat_count=1, batch_size=1): | |
| """ | |
| Creates tf.data.Dataset object. | |
| """ | |
| def _parse_function(filename, label): | |
| image_string = tf.read_file(filename) | |
| image = tf.image.decode_image(image_string, channels=3) | |
| image.set_shape([None, None, 3]) | |
| image = tf.image.resize_images(image, input_shape[:2]) | |
| image = tf.div(tf.subtract(image, 127.5), 127.5) # normalization | |
| return {'input_image': image}, label | |
| if labels is None: | |
| labels = [0]*len(filenames) | |
| labels=np.array(labels) | |
| # Expand the shape of "labels" if necessory | |
| if len(labels.shape) == 1: | |
| labels = np.expand_dims(labels, axis=1) | |
| filenames = tf.constant(filenames) | |
| labels = tf.constant(labels) | |
| labels = tf.cast(labels, tf.float32) | |
| dataset = tf.data.Dataset.from_tensor_slices((filenames, labels)) | |
| dataset = dataset.map(_parse_function) | |
| if perform_shuffle: | |
| # Randomizes input using a window of 256 elements (read into memory) | |
| dataset = dataset.shuffle(buffer_size=256) | |
| dataset = dataset.repeat(repeat_count) # Repeats dataset this # times | |
| dataset = dataset.batch(batch_size) # Batch size to use | |
| iterator = dataset.make_one_shot_iterator() | |
| batch_features, batch_labels = iterator.get_next() | |
| return batch_features, batch_labels | |
| def train(test_files, test_labels, experiment_folder): | |
| from tensorflow.python.keras.models import Model | |
| from tensorflow.python.keras.layers import Dense, Flatten | |
| from tensorflow.python.keras.layers import Input | |
| from tensorflow.python.keras.optimizers import Adam | |
| inp = Input(input_shape, name='input_image') | |
| out = Flatten()(inp) | |
| out = Dense(1)(out) | |
| model = Model(inputs=[inp], outputs=out) | |
| model.compile(optimizer=Adam(lr=0.001), loss='binary_crossentropy') | |
| est_catvsdog = tf.keras.estimator.model_to_estimator(keras_model=model, | |
| model_dir=experiment_folder) | |
| train_input_fn = lambda: imgs_input_fn(test_files, test_labels, | |
| perform_shuffle=True, repeat_count=1, batch_size=1) | |
| train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn, max_steps=1) | |
| eval_input_fn = lambda: imgs_input_fn(test_files, labels=test_labels, perform_shuffle=False) | |
| eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn) | |
| tf.estimator.train_and_evaluate(est_catvsdog, train_spec, eval_spec) | |
| def main(): | |
| filenames = ['/input/in.jpg'] | |
| labels = [0] | |
| train(filenames, labels, experiment_folder) | |
| if '__main__' in __name__: | |
| main() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| python example.py | |
| 2018-07-25 17:31:27.940345: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA | |
| 2018-07-25 17:31:28.021305: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:892] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero | |
| 2018-07-25 17:31:28.021579: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Found device 0 with properties: | |
| name: GeForce GTX 1080 Ti major: 6 minor: 1 memoryClockRate(GHz): 1.582 | |
| pciBusID: 0000:01:00.0 | |
| totalMemory: 10.91GiB freeMemory: 957.38MiB | |
| 2018-07-25 17:31:28.021591: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1120] Creating TensorFlow device (/device:GPU:0) -> (device: 0, name: GeForce GTX 1080 Ti, pci bus id: 0000:01:00.0, compute capability: 6.1) | |
| 2018-07-25 17:31:28.248544: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1120] Creating TensorFlow device (/device:GPU:0) -> (device: 0, name: GeForce GTX 1080 Ti, pci bus id: 0000:01:00.0, compute capability: 6.1) | |
| ERROR:tensorflow:Couldn't match files for checkpoint /output// | |
| 2018-07-25 17:31:28.687087: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1120] Creating TensorFlow device (/device:GPU:0) -> (device: 0, name: GeForce GTX 1080 Ti, pci bus id: 0000:01:00.0, compute capability: 6.1) | |
| 2018-07-25 17:31:28.708788: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| 2018-07-25 17:31:28.709210: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| 2018-07-25 17:31:28.710153: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| 2018-07-25 17:31:28.711168: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| 2018-07-25 17:31:28.711350: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| 2018-07-25 17:31:28.711580: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| 2018-07-25 17:31:28.712709: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| 2018-07-25 17:31:28.712917: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| 2018-07-25 17:31:28.713439: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| 2018-07-25 17:31:28.713741: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| 2018-07-25 17:31:28.714130: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| 2018-07-25 17:31:28.714436: W tensorflow/core/framework/op_kernel.cc:1192] Not found: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| Traceback (most recent call last): | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1323, in _do_call | |
| return fn(*args) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1302, in _run_fn | |
| status, run_metadata) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py", line 473, in __exit__ | |
| c_api.TF_GetCode(self.status.status)) | |
| tensorflow.python.framework.errors_impl.NotFoundError: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| [[Node: save/RestoreV2_1 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:localhost/replica:0/task:0/device:CPU:0"](_arg_save/Const_0_0, save/RestoreV2_1/tensor_names, save/RestoreV2_1/shape_and_slices)]] | |
| [[Node: save/RestoreV2_11/_1 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device_incarnation=1, tensor_name="edge_28_save/RestoreV2_11", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:GPU:0"]()]] | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/home/roman/Dev/avatars/example.py", line 76, in <module> | |
| main() | |
| File "/home/roman/Dev/avatars/example.py", line 72, in main | |
| train(filenames, labels, experiment_folder) | |
| File "/home/roman/Dev/avatars/example.py", line 66, in train | |
| tf.estimator.train_and_evaluate(est_catvsdog, train_spec, eval_spec) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/estimator/training.py", line 432, in train_and_evaluate | |
| executor.run_local() | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/estimator/training.py", line 611, in run_local | |
| hooks=train_hooks) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/estimator/estimator.py", line 302, in train | |
| loss = self._train_model(input_fn, hooks, saving_listeners) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/estimator/estimator.py", line 780, in _train_model | |
| log_step_count_steps=self._config.log_step_count_steps) as mon_sess: | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 368, in MonitoredTrainingSession | |
| stop_grace_period_secs=stop_grace_period_secs) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 673, in __init__ | |
| stop_grace_period_secs=stop_grace_period_secs) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 493, in __init__ | |
| self._sess = _RecoverableSession(self._coordinated_creator) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 851, in __init__ | |
| _WrappedSession.__init__(self, self._create_session()) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 856, in _create_session | |
| return self._sess_creator.create_session() | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 554, in create_session | |
| self.tf_sess = self._session_creator.create_session() | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 428, in create_session | |
| init_fn=self._scaffold.init_fn) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/session_manager.py", line 273, in prepare_session | |
| config=config) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/session_manager.py", line 205, in _restore_checkpoint | |
| saver.restore(sess, ckpt.model_checkpoint_path) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/saver.py", line 1666, in restore | |
| {self.saver_def.filename_tensor_name: save_path}) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 889, in run | |
| run_metadata_ptr) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1120, in _run | |
| feed_dict_tensor, options, run_metadata) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1317, in _do_run | |
| options, run_metadata) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1336, in _do_call | |
| raise type(e)(node_def, op, message) | |
| tensorflow.python.framework.errors_impl.NotFoundError: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| [[Node: save/RestoreV2_1 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:localhost/replica:0/task:0/device:CPU:0"](_arg_save/Const_0_0, save/RestoreV2_1/tensor_names, save/RestoreV2_1/shape_and_slices)]] | |
| [[Node: save/RestoreV2_11/_1 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device_incarnation=1, tensor_name="edge_28_save/RestoreV2_11", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:GPU:0"]()]] | |
| Caused by op 'save/RestoreV2_1', defined at: | |
| File "/home/roman/Dev/avatars/example.py", line 76, in <module> | |
| main() | |
| File "/home/roman/Dev/avatars/example.py", line 72, in main | |
| train(filenames, labels, experiment_folder) | |
| File "/home/roman/Dev/avatars/example.py", line 66, in train | |
| tf.estimator.train_and_evaluate(est_catvsdog, train_spec, eval_spec) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/estimator/training.py", line 432, in train_and_evaluate | |
| executor.run_local() | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/estimator/training.py", line 611, in run_local | |
| hooks=train_hooks) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/estimator/estimator.py", line 302, in train | |
| loss = self._train_model(input_fn, hooks, saving_listeners) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/estimator/estimator.py", line 780, in _train_model | |
| log_step_count_steps=self._config.log_step_count_steps) as mon_sess: | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 368, in MonitoredTrainingSession | |
| stop_grace_period_secs=stop_grace_period_secs) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 673, in __init__ | |
| stop_grace_period_secs=stop_grace_period_secs) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 493, in __init__ | |
| self._sess = _RecoverableSession(self._coordinated_creator) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 851, in __init__ | |
| _WrappedSession.__init__(self, self._create_session()) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 856, in _create_session | |
| return self._sess_creator.create_session() | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 554, in create_session | |
| self.tf_sess = self._session_creator.create_session() | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 419, in create_session | |
| self._scaffold.finalize() | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/monitored_session.py", line 212, in finalize | |
| self._saver.build() | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/saver.py", line 1227, in build | |
| self._build(self._filename, build_save=True, build_restore=True) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/saver.py", line 1263, in _build | |
| build_save=build_save, build_restore=build_restore) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/saver.py", line 745, in _build_internal | |
| restore_sequentially, reshape) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/saver.py", line 470, in _AddShardedRestoreOps | |
| name="restore_shard")) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/saver.py", line 427, in _AddRestoreOps | |
| tensors = self.restore_op(filename_tensor, saveable, preferred_shard) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/training/saver.py", line 267, in restore_op | |
| [spec.tensor.dtype])[0]) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/ops/gen_io_ops.py", line 1021, in restore_v2 | |
| shape_and_slices=shape_and_slices, dtypes=dtypes, name=name) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper | |
| op_def=op_def) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 2956, in create_op | |
| op_def=op_def) | |
| File "/home/roman/Dev/venv/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 1470, in __init__ | |
| self._traceback = self._graph._extract_stack() # pylint: disable=protected-access | |
| NotFoundError (see above for traceback): Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /output// | |
| [[Node: save/RestoreV2_1 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:localhost/replica:0/task:0/device:CPU:0"](_arg_save/Const_0_0, save/RestoreV2_1/tensor_names, save/RestoreV2_1/shape_and_slices)]] | |
| [[Node: save/RestoreV2_11/_1 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device_incarnation=1, tensor_name="edge_28_save/RestoreV2_11", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:GPU:0"]()]] | |
| Process finished with exit code 1 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment