Created
August 16, 2024 19:45
-
-
Save justinchuby/79ecd112eeeb9c6402651a6a2b1d6813 to your computer and use it in GitHub Desktop.
Preserve ops error
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] failed while attempting to run meta for aten.group_norm.default | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] Traceback (most recent call last): | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py", line 1971, in _dispatch_impl | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] r = func(*args, **kwargs) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_ops.py", line 713, in __call__ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] return self._op(*args, **kwargs) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/export/exported_program.py", line 134, in _register_cia_to_meta | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] return kernel._op_dk( | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 3154, in native_group_norm | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] out, mean, rstd = _normalize(input_reshaped, reduction_dims, eps) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 3112, in _normalize | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] biased_var, mean = torch.var_mean( | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_decomp/__init__.py", line 88, in _fn | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] return f(*args, **kwargs, out=None if is_none else out_kwargs) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_prims_common/wrappers.py", line 269, in _fn | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] result = fn(*args, **kwargs) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 2545, in var_mean | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] m = mean(a, dim, keepdim) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 2497, in mean | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] result = true_divide(result, nelem) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_prims_common/wrappers.py", line 269, in _fn | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] result = fn(*args, **kwargs) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_prims_common/wrappers.py", line 141, in _fn | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] result = fn(**bound.arguments) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 1050, in _ref | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] output = prim(a, b) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 1753, in true_divide | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] return prims.div(a, b) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_ops.py", line 713, in __call__ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] return self._op(*args, **kwargs) | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/export/exported_program.py", line 130, in _register_cia_to_meta | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] assert torch._C._dispatch_has_kernel_for_dispatch_key( | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
E0816 19:43:22.023000 57436 torch/_subclasses/fake_tensor.py:1975] AssertionError | |
[torch.onnx] Translate the graph into ONNX... ❌ | |
Traceback (most recent call last): | |
File "/home/justinchu/dev/torch-onnx/src/torch_onnx/_core.py", line 1077, in export | |
decomposed_program = _prepare_exported_program_for_export( | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/dev/torch-onnx/src/torch_onnx/_core.py", line 714, in _prepare_exported_program_for_export | |
exported_program = _fx_passes.decompose_with_registry(exported_program, registry) | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/dev/torch-onnx/src/torch_onnx/_fx_passes.py", line 45, in decompose_with_registry | |
return exported_program.run_decompositions( | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/export/exported_program.py", line 96, in wrapper | |
return fn(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/export/exported_program.py", line 1023, in run_decompositions | |
return _decompose_exported_program( | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/export/exported_program.py", line 640, in _decompose_exported_program | |
gm, new_graph_signature = _decompose_and_get_gm_with_new_signature_constants( | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/export/exported_program.py", line 487, in _decompose_and_get_gm_with_new_signature_constants | |
gm, graph_signature = aot_export_module( | |
^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 1241, in aot_export_module | |
fx_g, metadata, in_spec, out_spec = _aot_export_function( | |
^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 1475, in _aot_export_function | |
fx_g, meta = create_aot_dispatcher_function( | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 521, in create_aot_dispatcher_function | |
return _create_aot_dispatcher_function( | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 622, in _create_aot_dispatcher_function | |
fw_metadata = run_functionalized_fw_and_collect_metadata( | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/collect_metadata_analysis.py", line 168, in inner | |
flat_f_outs = f(*flat_f_args) | |
^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 179, in flat_fn | |
tree_out = fn(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/traced_function_transforms.py", line 805, in functional_call | |
out = PropagateUnbackedSymInts(mod).run( | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/fx/interpreter.py", line 147, in run | |
self.env[node] = self.run_node(node) | |
^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/fx/experimental/symbolic_shapes.py", line 5592, in run_node | |
result = super().run_node(n) | |
^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/fx/interpreter.py", line 204, in run_node | |
return getattr(self, n.op)(n.target, args, kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/fx/interpreter.py", line 276, in call_function | |
return target(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_ops.py", line 713, in __call__ | |
return self._op(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_higher_order_ops/utils.py", line 64, in inner | |
return autograd_not_implemented_inner(op, deferred_error, *args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_higher_order_ops/utils.py", line 37, in autograd_not_implemented_inner | |
result = operator(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_ops.py", line 713, in __call__ | |
return self._op(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_subclasses/functional_tensor.py", line 480, in __torch_dispatch__ | |
outs_unwrapped = func._op_dk( | |
^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/utils/_stats.py", line 21, in wrapper | |
return fn(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py", line 1201, in __torch_dispatch__ | |
return self.dispatch(func, types, args, kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py", line 1653, in dispatch | |
return self._cached_dispatch_impl(func, types, args, kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py", line 1300, in _cached_dispatch_impl | |
output = self._dispatch_impl(func, types, args, kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_subclasses/fake_tensor.py", line 1971, in _dispatch_impl | |
r = func(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_ops.py", line 713, in __call__ | |
return self._op(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/export/exported_program.py", line 134, in _register_cia_to_meta | |
return kernel._op_dk( | |
^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 3154, in native_group_norm | |
out, mean, rstd = _normalize(input_reshaped, reduction_dims, eps) | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 3112, in _normalize | |
biased_var, mean = torch.var_mean( | |
^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_decomp/__init__.py", line 88, in _fn | |
return f(*args, **kwargs, out=None if is_none else out_kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_prims_common/wrappers.py", line 269, in _fn | |
result = fn(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 2545, in var_mean | |
m = mean(a, dim, keepdim) | |
^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 2497, in mean | |
result = true_divide(result, nelem) | |
^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_prims_common/wrappers.py", line 269, in _fn | |
result = fn(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_prims_common/wrappers.py", line 141, in _fn | |
result = fn(**bound.arguments) | |
^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 1050, in _ref | |
output = prim(a, b) | |
^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_refs/__init__.py", line 1753, in true_divide | |
return prims.div(a, b) | |
^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/_ops.py", line 713, in __call__ | |
return self._op(*args, **kwargs) | |
^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/torch/export/exported_program.py", line 130, in _register_cia_to_meta | |
assert torch._C._dispatch_has_kernel_for_dispatch_key( | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
AssertionError: While executing %group_norm : [num_users=1] = call_function[target=torch.ops.aten.group_norm.default](args = (%conv3d, 8, %p_getattr_l__self___down_layers_0___1___norm1_weight, %p_getattr_l__self___down_layers_0___1___norm1_bias), kwargs = {}) | |
Original traceback: | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/monai/networks/nets/segresnet.py", line 180, in forward | |
x, down_x = self.encode(x) | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/monai/networks/nets/segresnet.py", line 164, in encode | |
x = down(x) | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/monai/networks/blocks/segresnet_block.py", line 86, in forward | |
x = self.norm1(x) | |
The above exception was the direct cause of the following exception: | |
Traceback (most recent call last): | |
File "/home/justinchu/dev/torch-onnx/tests/models/upsample.py", line 26, in <module> | |
main() | |
File "/home/justinchu/dev/torch-onnx/tests/models/upsample.py", line 21, in main | |
program = torch.onnx.export(model, (data,)) | |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
File "/home/justinchu/dev/torch-onnx/src/torch_onnx/_patch.py", line 167, in _torch_onnx_export | |
onnx_program = _core.export( | |
^^^^^^^^^^^^^ | |
File "/home/justinchu/dev/torch-onnx/src/torch_onnx/_core.py", line 1105, in export | |
raise errors.OnnxConversionError( | |
torch_onnx.errors.OnnxConversionError: Failed to convert the exported program to an ONNX model. This is step 2/2 of exporting the model to ONNX. Next steps: | |
- If there is a missing ONNX function, implement it and register it to the registry. | |
- If there is an internal error during ONNX conversion, debug the error and summit a PR to PyTorch. | |
- Save the ExportedProgram as a pt2 file and create an error report with `export(..., report=True)`. Create an issue in the PyTorch GitHub repository against the *onnx* component. Attach the pt2 model and the error report. | |
Error report has been saved to 'onnx_export_2024-08-16_19-43-19-065144_conversion.md'. | |
## Exception summary | |
<class 'AssertionError'>: While executing %group_norm : [num_users=1] = call_function[target=torch.ops.aten.group_norm.default](args = (%conv3d, 8, %p_getattr_l__self___down_layers_0___1___norm1_weight, %p_getattr_l__self___down_layers_0___1___norm1_bias), kwargs = {}) | |
Original traceback: | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/monai/networks/nets/segresnet.py", line 180, in forward | |
x, down_x = self.encode(x) | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/monai/networks/nets/segresnet.py", line 164, in encode | |
x = down(x) | |
File "/home/justinchu/anaconda3/envs/onnx/lib/python3.11/site-packages/monai/networks/blocks/segresnet_block.py", line 86, in forward | |
x = self.norm1(x) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment