Created
March 14, 2018 08:27
-
-
Save TopHatCroat/768cd06704d21e8a22a1b155d3ff8872 to your computer and use it in GitHub Desktop.
error.txt
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| WARNING:tensorflow:From /home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/contrib/slim/python/slim/learning.py:736: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. | |
| Instructions for updating: | |
| Please switch to tf.train.MonitoredTrainingSession | |
| 2018-03-14 00:07:43.873721: I tensorflow/core/platform/cpu_feature_guard.cc:140] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA | |
| INFO:tensorflow:Restoring parameters from /home/antonio/Development/TensorFlow/tuts/obj_detect/models/checkpoint/model.ckpt | |
| INFO:tensorflow:Running local_init_op. | |
| INFO:tensorflow:Done running local_init_op. | |
| INFO:tensorflow:Starting Session. | |
| INFO:tensorflow:Saving checkpoint to path /home/antonio/Development/TensorFlow/tuts/obj_detect/models/model/train/model.ckpt | |
| INFO:tensorflow:Starting Queues. | |
| INFO:tensorflow:global_step/sec: 0 | |
| INFO:tensorflow:Error reported to Coordinator: Reduction axis 1 is empty in shape [9,0] | |
| [[Node: Loss/RPNLoss/Match/cond/ArgMax_1 = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](Loss/RPNLoss/Match/cond/Shape_1/Switch:1, Loss/RPNLoss/Match/cond/ArgMax_1/dimension)]] | |
| Caused by op 'Loss/RPNLoss/Match/cond/ArgMax_1', defined at: | |
| File "object_detection/train.py", line 167, in <module> | |
| tf.app.run() | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 126, in run | |
| _sys.exit(main(argv)) | |
| File "object_detection/train.py", line 163, in main | |
| worker_job_name, is_chief, FLAGS.train_dir) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/trainer.py", line 246, in train | |
| clones = model_deploy.create_clones(deploy_config, model_fn, [input_queue]) | |
| File "/home/antonio/Development/TensorFlow/models/research/slim/deployment/model_deploy.py", line 193, in create_clones | |
| outputs = model_fn(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/trainer.py", line 181, in _create_losses | |
| losses_dict = detection_model.loss(prediction_dict, true_image_shapes) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py", line 1567, in loss | |
| groundtruth_classes_with_background_list) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py", line 1621, in _loss_rpn | |
| groundtruth_boxlists, len(groundtruth_boxlists)*[None]) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/target_assigner.py", line 447, in batch_assign_targets | |
| anchors, gt_boxes, gt_class_targets, gt_weights) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/target_assigner.py", line 162, in assign | |
| match = self._matcher.match(match_quality_matrix, **params) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/matcher.py", line 239, in match | |
| return Match(self._match(similarity_matrix, **params), | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/matchers/argmax_matcher.py", line 190, in _match | |
| _match_when_rows_are_non_empty, _match_when_rows_are_empty) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 432, in new_func | |
| return func(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2018, in cond | |
| orig_res_t, res_t = context_t.BuildCondBranch(true_fn) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1868, in BuildCondBranch | |
| original_result = fn() | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/matchers/argmax_matcher.py", line 169, in _match_when_rows_are_non_empty | |
| output_type=tf.int32) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 432, in new_func | |
| return func(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 211, in argmax | |
| return gen_math_ops.arg_max(input, axis, name=name, output_type=output_type) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 472, in arg_max | |
| name=name) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper | |
| op_def=op_def) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3271, in create_op | |
| op_def=op_def) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1650, in __init__ | |
| self._traceback = self._graph._extract_stack() # pylint: disable=protected-access | |
| InvalidArgumentError (see above for traceback): Reduction axis 1 is empty in shape [9,0] | |
| [[Node: Loss/RPNLoss/Match/cond/ArgMax_1 = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](Loss/RPNLoss/Match/cond/Shape_1/Switch:1, Loss/RPNLoss/Match/cond/ArgMax_1/dimension)]] | |
| Traceback (most recent call last): | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1361, in _do_call | |
| return fn(*args) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1340, in _run_fn | |
| target_list, status, run_metadata) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py", line 516, in __exit__ | |
| c_api.TF_GetCode(self.status.status)) | |
| tensorflow.python.framework.errors_impl.InvalidArgumentError: Reduction axis 1 is empty in shape [9,0] | |
| [[Node: Loss/RPNLoss/Match/cond/ArgMax_1 = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](Loss/RPNLoss/Match/cond/Shape_1/Switch:1, Loss/RPNLoss/Match/cond/ArgMax_1/dimension)]] | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/training/coordinator.py", line 297, in stop_on_exception | |
| yield | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/training/coordinator.py", line 495, in run | |
| self.run_loop() | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/training/supervisor.py", line 1030, in run_loop | |
| self._sv.global_step]) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 905, in run | |
| run_metadata_ptr) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1137, in _run | |
| feed_dict_tensor, options, run_metadata) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1355, in _do_run | |
| options, run_metadata) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1374, in _do_call | |
| raise type(e)(node_def, op, message) | |
| tensorflow.python.framework.errors_impl.InvalidArgumentError: Reduction axis 1 is empty in shape [9,0] | |
| [[Node: Loss/RPNLoss/Match/cond/ArgMax_1 = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](Loss/RPNLoss/Match/cond/Shape_1/Switch:1, Loss/RPNLoss/Match/cond/ArgMax_1/dimension)]] | |
| Caused by op 'Loss/RPNLoss/Match/cond/ArgMax_1', defined at: | |
| File "object_detection/train.py", line 167, in <module> | |
| tf.app.run() | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 126, in run | |
| _sys.exit(main(argv)) | |
| File "object_detection/train.py", line 163, in main | |
| worker_job_name, is_chief, FLAGS.train_dir) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/trainer.py", line 246, in train | |
| clones = model_deploy.create_clones(deploy_config, model_fn, [input_queue]) | |
| File "/home/antonio/Development/TensorFlow/models/research/slim/deployment/model_deploy.py", line 193, in create_clones | |
| outputs = model_fn(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/trainer.py", line 181, in _create_losses | |
| losses_dict = detection_model.loss(prediction_dict, true_image_shapes) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py", line 1567, in loss | |
| groundtruth_classes_with_background_list) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py", line 1621, in _loss_rpn | |
| groundtruth_boxlists, len(groundtruth_boxlists)*[None]) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/target_assigner.py", line 447, in batch_assign_targets | |
| anchors, gt_boxes, gt_class_targets, gt_weights) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/target_assigner.py", line 162, in assign | |
| match = self._matcher.match(match_quality_matrix, **params) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/matcher.py", line 239, in match | |
| return Match(self._match(similarity_matrix, **params), | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/matchers/argmax_matcher.py", line 190, in _match | |
| _match_when_rows_are_non_empty, _match_when_rows_are_empty) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 432, in new_func | |
| return func(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2018, in cond | |
| orig_res_t, res_t = context_t.BuildCondBranch(true_fn) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1868, in BuildCondBranch | |
| original_result = fn() | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/matchers/argmax_matcher.py", line 169, in _match_when_rows_are_non_empty | |
| output_type=tf.int32) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 432, in new_func | |
| return func(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 211, in argmax | |
| return gen_math_ops.arg_max(input, axis, name=name, output_type=output_type) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 472, in arg_max | |
| name=name) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper | |
| op_def=op_def) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3271, in create_op | |
| op_def=op_def) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1650, in __init__ | |
| self._traceback = self._graph._extract_stack() # pylint: disable=protected-access | |
| InvalidArgumentError (see above for traceback): Reduction axis 1 is empty in shape [9,0] | |
| [[Node: Loss/RPNLoss/Match/cond/ArgMax_1 = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](Loss/RPNLoss/Match/cond/Shape_1/Switch:1, Loss/RPNLoss/Match/cond/ArgMax_1/dimension)]] | |
| Traceback (most recent call last): | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1361, in _do_call | |
| return fn(*args) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1340, in _run_fn | |
| target_list, status, run_metadata) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py", line 516, in __exit__ | |
| c_api.TF_GetCode(self.status.status)) | |
| tensorflow.python.framework.errors_impl.InvalidArgumentError: Reduction axis 1 is empty in shape [9,0] | |
| [[Node: Loss/RPNLoss/Match/cond/ArgMax_1 = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](Loss/RPNLoss/Match/cond/Shape_1/Switch:1, Loss/RPNLoss/Match/cond/ArgMax_1/dimension)]] | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/training/supervisor.py", line 990, in managed_session | |
| yield sess | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/contrib/slim/python/slim/learning.py", line 768, in train | |
| sess, train_op, global_step, train_step_kwargs) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/contrib/slim/python/slim/learning.py", line 487, in train_step | |
| run_metadata=run_metadata) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 905, in run | |
| run_metadata_ptr) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1137, in _run | |
| feed_dict_tensor, options, run_metadata) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1355, in _do_run | |
| options, run_metadata) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1374, in _do_call | |
| raise type(e)(node_def, op, message) | |
| tensorflow.python.framework.errors_impl.InvalidArgumentError: Reduction axis 1 is empty in shape [9,0] | |
| [[Node: Loss/RPNLoss/Match/cond/ArgMax_1 = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](Loss/RPNLoss/Match/cond/Shape_1/Switch:1, Loss/RPNLoss/Match/cond/ArgMax_1/dimension)]] | |
| Caused by op 'Loss/RPNLoss/Match/cond/ArgMax_1', defined at: | |
| File "object_detection/train.py", line 167, in <module> | |
| tf.app.run() | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 126, in run | |
| _sys.exit(main(argv)) | |
| File "object_detection/train.py", line 163, in main | |
| worker_job_name, is_chief, FLAGS.train_dir) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/trainer.py", line 246, in train | |
| clones = model_deploy.create_clones(deploy_config, model_fn, [input_queue]) | |
| File "/home/antonio/Development/TensorFlow/models/research/slim/deployment/model_deploy.py", line 193, in create_clones | |
| outputs = model_fn(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/trainer.py", line 181, in _create_losses | |
| losses_dict = detection_model.loss(prediction_dict, true_image_shapes) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py", line 1567, in loss | |
| groundtruth_classes_with_background_list) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py", line 1621, in _loss_rpn | |
| groundtruth_boxlists, len(groundtruth_boxlists)*[None]) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/target_assigner.py", line 447, in batch_assign_targets | |
| anchors, gt_boxes, gt_class_targets, gt_weights) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/target_assigner.py", line 162, in assign | |
| match = self._matcher.match(match_quality_matrix, **params) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/matcher.py", line 239, in match | |
| return Match(self._match(similarity_matrix, **params), | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/matchers/argmax_matcher.py", line 190, in _match | |
| _match_when_rows_are_non_empty, _match_when_rows_are_empty) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 432, in new_func | |
| return func(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2018, in cond | |
| orig_res_t, res_t = context_t.BuildCondBranch(true_fn) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1868, in BuildCondBranch | |
| original_result = fn() | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/matchers/argmax_matcher.py", line 169, in _match_when_rows_are_non_empty | |
| output_type=tf.int32) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 432, in new_func | |
| return func(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 211, in argmax | |
| return gen_math_ops.arg_max(input, axis, name=name, output_type=output_type) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 472, in arg_max | |
| name=name) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper | |
| op_def=op_def) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3271, in create_op | |
| op_def=op_def) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1650, in __init__ | |
| self._traceback = self._graph._extract_stack() # pylint: disable=protected-access | |
| InvalidArgumentError (see above for traceback): Reduction axis 1 is empty in shape [9,0] | |
| [[Node: Loss/RPNLoss/Match/cond/ArgMax_1 = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](Loss/RPNLoss/Match/cond/Shape_1/Switch:1, Loss/RPNLoss/Match/cond/ArgMax_1/dimension)]] | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "object_detection/train.py", line 167, in <module> | |
| tf.app.run() | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 126, in run | |
| _sys.exit(main(argv)) | |
| File "object_detection/train.py", line 163, in main | |
| worker_job_name, is_chief, FLAGS.train_dir) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/trainer.py", line 360, in train | |
| saver=saver) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/contrib/slim/python/slim/learning.py", line 783, in train | |
| ignore_live_threads=ignore_live_threads) | |
| File "/usr/lib64/python3.6/contextlib.py", line 99, in __exit__ | |
| self.gen.throw(type, value, traceback) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/training/supervisor.py", line 1000, in managed_session | |
| self.stop(close_summary_writer=close_summary_writer) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/training/supervisor.py", line 828, in stop | |
| ignore_live_threads=ignore_live_threads) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/training/coordinator.py", line 389, in join | |
| six.reraise(*self._exc_info_to_raise) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/six.py", line 693, in reraise | |
| raise value | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/training/coordinator.py", line 297, in stop_on_exception | |
| yield | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/training/coordinator.py", line 495, in run | |
| self.run_loop() | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/training/supervisor.py", line 1030, in run_loop | |
| self._sv.global_step]) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 905, in run | |
| run_metadata_ptr) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1137, in _run | |
| feed_dict_tensor, options, run_metadata) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1355, in _do_run | |
| options, run_metadata) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1374, in _do_call | |
| raise type(e)(node_def, op, message) | |
| tensorflow.python.framework.errors_impl.InvalidArgumentError: Reduction axis 1 is empty in shape [9,0] | |
| [[Node: Loss/RPNLoss/Match/cond/ArgMax_1 = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](Loss/RPNLoss/Match/cond/Shape_1/Switch:1, Loss/RPNLoss/Match/cond/ArgMax_1/dimension)]] | |
| Caused by op 'Loss/RPNLoss/Match/cond/ArgMax_1', defined at: | |
| File "object_detection/train.py", line 167, in <module> | |
| tf.app.run() | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 126, in run | |
| _sys.exit(main(argv)) | |
| File "object_detection/train.py", line 163, in main | |
| worker_job_name, is_chief, FLAGS.train_dir) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/trainer.py", line 246, in train | |
| clones = model_deploy.create_clones(deploy_config, model_fn, [input_queue]) | |
| File "/home/antonio/Development/TensorFlow/models/research/slim/deployment/model_deploy.py", line 193, in create_clones | |
| outputs = model_fn(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/trainer.py", line 181, in _create_losses | |
| losses_dict = detection_model.loss(prediction_dict, true_image_shapes) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py", line 1567, in loss | |
| groundtruth_classes_with_background_list) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py", line 1621, in _loss_rpn | |
| groundtruth_boxlists, len(groundtruth_boxlists)*[None]) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/target_assigner.py", line 447, in batch_assign_targets | |
| anchors, gt_boxes, gt_class_targets, gt_weights) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/target_assigner.py", line 162, in assign | |
| match = self._matcher.match(match_quality_matrix, **params) | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/core/matcher.py", line 239, in match | |
| return Match(self._match(similarity_matrix, **params), | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/matchers/argmax_matcher.py", line 190, in _match | |
| _match_when_rows_are_non_empty, _match_when_rows_are_empty) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 432, in new_func | |
| return func(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2018, in cond | |
| orig_res_t, res_t = context_t.BuildCondBranch(true_fn) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1868, in BuildCondBranch | |
| original_result = fn() | |
| File "/home/antonio/Development/TensorFlow/models/research/object_detection/matchers/argmax_matcher.py", line 169, in _match_when_rows_are_non_empty | |
| output_type=tf.int32) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 432, in new_func | |
| return func(*args, **kwargs) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 211, in argmax | |
| return gen_math_ops.arg_max(input, axis, name=name, output_type=output_type) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 472, in arg_max | |
| name=name) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper | |
| op_def=op_def) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3271, in create_op | |
| op_def=op_def) | |
| File "/home/antonio/Development/TensorFlow/env/ccfoto/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1650, in __init__ | |
| self._traceback = self._graph._extract_stack() # pylint: disable=protected-access | |
| InvalidArgumentError (see above for traceback): Reduction axis 1 is empty in shape [9,0] | |
| [[Node: Loss/RPNLoss/Match/cond/ArgMax_1 = ArgMax[T=DT_FLOAT, Tidx=DT_INT32, output_type=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](Loss/RPNLoss/Match/cond/Shape_1/Switch:1, Loss/RPNLoss/Match/cond/ArgMax_1/dimension)]] | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment