rpn_error.txt

Traceback (most recent call last):
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1335, in _do_call
    return fn(*args)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1320, in _run_fn
    options, feed_dict, fetch_list, target_list, run_metadata)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1408, in _call_tf_sessionrun
    run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Shapes of all inputs must match: values[0].shape = [4,200,304,3] != values[1].shape = [4,100,152,3]
	 [[{{node Sum_5/input}}]]
	 [[gradients/group1/block0/conv1/Conv2D_grad/Conv2DBackpropFilter/_3987]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/tensorpack-mask-rcnn/MaskRCNN/train.py", line 593, in <module>
    launch_train_with_config(traincfg, trainer)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/train/interface.py", line 94, in launch_train_with_config
    extra_callbacks=config.extra_callbacks)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/train/base.py", line 343, in train_with_defaults
    steps_per_epoch, starting_epoch, max_epoch)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/train/base.py", line 315, in train
    self.main_loop(steps_per_epoch, starting_epoch, max_epoch)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/utils/argtools.py", line 176, in wrapper
    return func(*args, **kwargs)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/train/base.py", line 280, in main_loop
    self.run_step()  # implemented by subclass
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/train/base.py", line 180, in run_step
    self.hooked_sess.run(self.train_op)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 693, in run
    run_metadata=run_metadata)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1188, in run
    run_metadata=run_metadata)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1287, in run
    raise six.reraise(*original_exc_info)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/six.py", line 693, in reraise
    raise value
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1272, in run
    return self._sess.run(*args, **kwargs)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1344, in run
    run_metadata=run_metadata)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1108, in run
    return self._sess.run(*args, **kwargs)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 930, in run
    run_metadata_ptr)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1153, in _run
    feed_dict_tensor, options, run_metadata)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1329, in _do_run
    run_metadata)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1349, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Shapes of all inputs must match: values[0].shape = [4,200,304,3] != values[1].shape = [4,100,152,3]
	 [[node Sum_5/input (defined at /tensorpack-mask-rcnn/MaskRCNN/train.py:271) ]]
	 [[gradients/group1/block0/conv1/Conv2D_grad/Conv2DBackpropFilter/_3987]]

Errors may have originated from an input operation.
Input Source operations connected to node Sum_5/input:
 group1/block2/conv3/bn/variance/EMA (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/models/tflayer.py:88)
 group1/block2/conv3/bn/Reshape_2/shape (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/models/batch_norm.py:192)
 group0/block0/conv2/bn/gamma (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/tfutils/varreplace.py:91)
 Pad_1/paddings (defined at /tensorpack-mask-rcnn/MaskRCNN/basemodel.py:206)
 group3/block0/Pad/paddings (defined at /tensorpack-mask-rcnn/MaskRCNN/basemodel.py:144)
 fpn/upsample_lat5/Tensordot/free (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/models/pool.py:130)
 fpn/upsample_lat4/strided_slice/stack (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/models/shape_utils.py:26)
 fpn/upsample_lat4/stack/0 (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/models/pool.py:138)
 fpn/upsample_lat3/unpool_mat (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/models/pool.py:122)
 fpn/upsample_lat3/ExpandDims/dim (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/models/pool.py:128)
 fpn/upsample_lat3/ExpandDims_1/dim (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/models/pool.py:129)
 strided_slice_2/stack_2 (defined at /tensorpack-mask-rcnn/MaskRCNN/basemodel.py:203)
 fpn/upsample_lat5/transpose/perm (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/models/pool.py:135)
 fpn/upsample_lat5/mul/y (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/models/pool.py:109)
 fpn/upsample_lat5/mul_1/y (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/models/pool.py:110)
 QueueInput/input_queue (defined at /anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/input_source/input_source.py:219)
 strided_slice_1/stack (defined at /tensorpack-mask-rcnn/MaskRCNN/basemodel.py:202)
 rpn_1/transpose/perm (defined at /tensorpack-mask-rcnn/MaskRCNN/model_rpn.py:41)
 strided_slice/stack (defined at /tensorpack-mask-rcnn/MaskRCNN/basemodel.py:192)
 image_preprocess/Const (defined at /tensorpack-mask-rcnn/MaskRCNN/basemodel.py:103)
 transpose/perm (defined at /tensorpack-mask-rcnn/MaskRCNN/train.py:80)
 image_preprocess/Const_1 (defined at /tensorpack-mask-rcnn/MaskRCNN/basemodel.py:104)
 mul/y (defined at /tensorpack-mask-rcnn/MaskRCNN/basemodel.py:194)

Original stack trace for 'Sum_5/input':
  File "/tensorpack-mask-rcnn/MaskRCNN/train.py", line 593, in <module>
    launch_train_with_config(traincfg, trainer)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/train/interface.py", line 84, in launch_train_with_config
    model._build_graph_get_cost, model.get_optimizer)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/utils/argtools.py", line 176, in wrapper
    return func(*args, **kwargs)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/train/tower.py", line 214, in setup_graph
    train_callbacks = self._setup_graph(input, get_cost_fn, get_opt_fn)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/train/trainers.py", line 418, in _setup_graph
    grads = self._make_get_grad_fn(input, get_cost_fn, get_opt_fn)()
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/train/tower.py", line 266, in get_grad_fn
    return compute_grad_from_inputs(*inputs)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/train/tower.py", line 245, in compute_grad_from_inputs
    cost = get_cost_fn(*inputs)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/tfutils/tower.py", line 286, in __call__
    output = self._tower_fn(*args)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorpack/graph_builder/model_desc.py", line 262, in _build_graph_get_cost
    ret = self.build_graph(*inputs)
  File "/tensorpack-mask-rcnn/MaskRCNN/train.py", line 132, in build_graph
    proposals, rpn_losses = self.rpn(images, features, anchor_inputs, inputs['orig_image_dims'])  # inputs?
  File "/tensorpack-mask-rcnn/MaskRCNN/train.py", line 271, in rpn
    losses = [tf.reduce_sum(multilevel_label_logits) + tf.reduce_sum( multilevel_box_logits)]
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 507, in new_func
    return func(*args, **kwargs)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 1288, in reduce_sum_v1
    return reduce_sum(input_tensor, axis, keepdims, name)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/util/dispatch.py", line 180, in wrapper
    return target(*args, **kwargs)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 1336, in reduce_sum
    name=name))
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 9693, in _sum
    name=name)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 511, in _apply_op_helper
    preferred_dtype=default_dtype)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1183, in internal_convert_to_tensor
    ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 1109, in _autopacking_conversion_function
    return _autopacking_helper(v, dtype, name or "packed")
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 1061, in _autopacking_helper
    return gen_array_ops.pack(elems_as_tensors, name=scope)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py", line 5448, in pack
    "Pack", values=values, axis=axis, name=name)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 788, in _apply_op_helper
    op_def=op_def)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 507, in new_func
    return func(*args, **kwargs)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3473, in create_op
    op_def=op_def)
  File "/anaconda3/envs/tensorflow_p36_13rc1/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1958, in __init__
    self._traceback = tf_stack.extract_stack()