gistfile1.txt

ERROR:common:
Traceback (most recent call last):
  File "/data/users/mlazos/pytorch/benchmarks/dynamo/common.py", line 2216, in check_accuracy
    new_result = self.run_n_iterations(
  File "/data/users/mlazos/pytorch/benchmarks/dynamo/common.py", line 1930, in run_n_iterations
    model_iter_fn(mod, inputs, collect_outputs=False)
  File "/data/users/mlazos/pytorch/torch/_dynamo/eval_frame.py", line 658, in _fn
    return fn(*args, **kwargs)
  File "/data/users/mlazos/pytorch/benchmarks/dynamo/huggingface.py", line 531, in forward_and_backward_pass
    cloned_inputs = clone_inputs(inputs)
  File "/data/users/mlazos/pytorch/benchmarks/dynamo/huggingface.py", line 532, in torch_dynamo_resume_in_forward_and_backward_pass_at_531
    self.optimizer_zero_grad(mod)
  File "/data/users/mlazos/pytorch/benchmarks/dynamo/huggingface.py", line 536, in torch_dynamo_resume_in_forward_and_backward_pass_at_532
    self.grad_scaler.scale(loss).backward()
  File "/data/users/mlazos/pytorch/torch/_tensor.py", line 648, in backward
    torch.autograd.backward(
  File "/data/users/mlazos/pytorch/torch/autograd/__init__.py", line 353, in backward
    _engine_run_backward(
  File "/data/users/mlazos/pytorch/torch/autograd/graph.py", line 824, in _engine_run_backward
    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
  File "/data/users/mlazos/pytorch/torch/autograd/function.py", line 307, in apply
    return user_fn(self, *args)
  File "/data/users/mlazos/pytorch/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2154, in backward
    return impl_fn()
  File "/data/users/mlazos/pytorch/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2140, in impl_fn
    out = CompiledFunction._backward_impl(ctx, all_args)
  File "/data/users/mlazos/pytorch/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2262, in _backward_impl
    out = call_func_at_runtime_with_args(
  File "/data/users/mlazos/pytorch/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args
    out = normalize_as_list(f(args))
  File "/data/users/mlazos/pytorch/torch/_dynamo/eval_frame.py", line 850, in _fn
    return fn(*args, **kwargs)
  File "/data/users/mlazos/pytorch/torch/_inductor/output_code.py", line 558, in __call__
    return self.current_callable(inputs)
  File "/data/users/mlazos/pytorch/torch/_inductor/utils.py", line 2441, in run
    return model(new_inputs)
  File "/tmp/tmp24bt8w_7/hh/chh4hestpo3pdfvjzr4uogib3gltvgxs3ge2fnqzrddnkix7mzhc.py", line 2860, in call
    (buf22[0], buf22[1], buf22[2], buf22[3], buf22[4], buf22[5], buf22[6], buf22[7], buf22[8], buf22[9], buf22[10], buf22[11], buf22[12], buf22[13], buf22[14], buf22[15]) = ___backward_subgraph_0_post_graph(___backward_subgraph_0_post_graph_args)
  File "/tmp/tmp24bt8w_7/hh/chh4hestpo3pdfvjzr4uogib3gltvgxs3ge2fnqzrddnkix7mzhc.py", line 1956, in ___backward_subgraph_0_post_graph
    extern_kernels.mm(___backward_subgraph_0_post_graph_buf42, ___backward_subgraph_0_post_graph_permute_35, out=___backward_subgraph_0_post_graph_buf43)
RuntimeError: expected mat1 and mat2 to have the same dtype, but got: float != c10::Half