Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save wanchaol/aceb0a1e8d3a93c8853689730b0f709f to your computer and use it in GitHub Desktop.
Save wanchaol/aceb0a1e8d3a93c8853689730b0f709f to your computer and use it in GitHub Desktop.
loss value: tensor(7.5122, grad_fn=<_TransducerBackward>)
caffe2/aten/src/ATen/native/Normalization.cpp:311:18: runtime error: 6.17887e+39 is outside the range of representable values of type 'float'
#0 0x7f877131e37a in std::tuple<at::Tensor, at::Tensor, at::Tensor> at::native::batch_norm_backward_cpu_template<float>(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>)::'lambda'(long, long)::operator()(long, long) const::'lambda0'(float&, float const&)::operator()(float&, float const&) const caffe2/aten/src/ATen/native/Normalization.cpp:311
#1 0x7f877131ddc3 in void at::apply_op<std::tuple<at::Tensor, at::Tensor, at::Tensor> at::native::batch_norm_backward_cpu_template<float>(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>)::'lambda'(long, long)::operator()(long, long) const::'lambda0'(float&, float const&), at::strided_tensor_iter_fixed<float, 8>, at::strided_tensor_iter_fixed<float, 8> >(long, long, std::tuple<at::Tensor, at::Tensor, at::Tensor> at::native::batch_norm_backward_cpu_template<float>(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>)::'lambda'(long, long)::operator()(long, long) const::'lambda0'(float&, float const&) const&, at::strided_tensor_iter_fixed<float, 8>, at::strided_tensor_iter_fixed<float, 8>) caffe2/aten/src/ATen/CPUApplyUtils.h:326
#2 0x7f877131b7f7 in void at::CPU_tensor_apply2<float, float, std::tuple<at::Tensor, at::Tensor, at::Tensor> at::native::batch_norm_backward_cpu_template<float>(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>)::'lambda'(long, long)::operator()(long, long) const::'lambda0'(float&, float const&)>(at::Tensor, at::Tensor, std::tuple<at::Tensor, at::Tensor, at::Tensor> at::native::batch_norm_backward_cpu_template<float>(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>)::'lambda'(long, long)::operator()(long, long) const::'lambda0'(float&, float const&)) caffe2/aten/src/ATen/CPUApplyUtils.h:427
#3 0x7f8771318840 in std::tuple<at::Tensor, at::Tensor, at::Tensor> at::native::batch_norm_backward_cpu_template<float>(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>)::'lambda'(long, long)::operator()(long, long) const caffe2/aten/src/ATen/native/Normalization.cpp:310
#4 0x7f8771316bb5 in void at::parallel_for<std::tuple<at::Tensor, at::Tensor, at::Tensor> at::native::batch_norm_backward_cpu_template<float>(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>)::'lambda'(long, long)>(long, long, long, std::tuple<at::Tensor, at::Tensor, at::Tensor> at::native::batch_norm_backward_cpu_template<float>(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>)::'lambda'(long, long) const&) caffe2/aten/src/ATen/Parallel.h:71
#5 0x7f87713162f5 in std::tuple<at::Tensor, at::Tensor, at::Tensor> at::native::batch_norm_backward_cpu_template<float>(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>) caffe2/aten/src/ATen/native/Normalization.cpp:266
#6 0x7f877130c171 in at::native::batch_norm_backward_cpu(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>)::$_2::operator()() const::'lambda0'()::operator()() const caffe2/aten/src/ATen/native/Normalization.cpp:593
#7 0x7f87712e8976 in at::native::batch_norm_backward_cpu(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>)::$_2::operator()() const caffe2/aten/src/ATen/native/Normalization.cpp:593
#8 0x7f87712e7bb2 in at::native::batch_norm_backward_cpu(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>) caffe2/aten/src/ATen/native/Normalization.cpp:593
#9 0x7f87709b7f23 in at::CPUType::native_batch_norm_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>) const buck-out/dev/gen/caffe2/aten/gen_aten=CPUType.cpp/CPUType.cpp:22887
#10 0x7f87b805cd7f in torch::autograd::VariableType::native_batch_norm_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>) const::$_104::operator()() const buck-out/dev/gen/caffe2/generate-code=VariableType_1.cpp/VariableType_1.cpp:13790
#11 0x7f87b8058eb9 in torch::autograd::VariableType::native_batch_norm_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>) const buck-out/dev/gen/caffe2/generate-code=VariableType_1.cpp/VariableType_1.cpp:13788
#12 0x7f87b780366d in at::native_batch_norm_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, bool, double, std::array<bool, 3ul>) buck-out/dev/gen/caffe2/aten/generated-aten-headers-cpu#header-mode-symlink-tree-with-header-map,headers/ATen/Functions.h:4142
#13 0x7f87b78024cb in torch::autograd::generated::NativeBatchNormBackward::apply(std::vector<torch::autograd::Variable, std::allocator<torch::autograd::Variable> >&&) buck-out/dev/gen/caffe2/generate-code=Functions.cpp/Functions.cpp:3934
#14 0x7f87b9043c63 in torch::autograd::Function::operator()(std::vector<torch::autograd::Variable, std::allocator<torch::autograd::Variable> >&&) caffe2/torch/csrc/autograd/function.h:118
#15 0x7f87b903019f in torch::autograd::call_function(torch::autograd::FunctionTask&) caffe2/torch/csrc/autograd/engine.cpp:420
#16 0x7f87b902c82e in torch::autograd::Engine::evaluate_function(torch::autograd::FunctionTask&) caffe2/torch/csrc/autograd/engine.cpp:451
#17 0x7f87b902af25 in torch::autograd::Engine::thread_main(torch::autograd::GraphTask*) caffe2/torch/csrc/autograd/engine.cpp:277
#18 0x7f87b902a0db in torch::autograd::Engine::thread_init(int) caffe2/torch/csrc/autograd/engine.cpp:251
#19 0x7f87bd12f301 in torch::autograd::python::PythonEngine::thread_init(int) caffe2/torch/csrc/autograd/python_engine.cpp:39
#20 0x7f87b907356c in void std::_Mem_fn_base<void (torch::autograd::Engine::*)(int), true>::operator()<int, void>(torch::autograd::Engine*, int&&) const ../fbcode/third-party-buck/gcc-5-glibc-2.23/build/libgcc/include/c++/5.5.0/functional:600
#21 0x7f87b9073402 in void std::_Bind_simple<std::_Mem_fn<void (torch::autograd::Engine::*)(int)> (torch::autograd::Engine*, int)>::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) ../fbcode/third-party-buck/gcc-5-glibc-2.23/build/libgcc/include/c++/5.5.0/functional:1530
#22 0x7f87b9073374 in std::_Bind_simple<std::_Mem_fn<void (torch::autograd::Engine::*)(int)> (torch::autograd::Engine*, int)>::operator()() ../fbcode/third-party-buck/gcc-5-glibc-2.23/build/libgcc/include/c++/5.5.0/functional:1520
#23 0x7f87b9072ed8 in std::thread::_Impl<std::_Bind_simple<std::_Mem_fn<void (torch::autograd::Engine::*)(int)> (torch::autograd::Engine*, int)> >::_M_run() ../fbcode/third-party-buck/gcc-5-glibc-2.23/build/libgcc/include/c++/5.5.0/thread:115
#24 0x7f87dc8c4190 in execute_native_thread_routine /home/engshare/third-party2/libgcc/5.x/src/gcc-5/libstdc++-v3/src/c++11/thread.cc:84
#25 0x7f87dd6717c8 in start_thread /home/engshare/third-party2/glibc/2.23/src/glibc-2.23/nptl/pthread_create.c:333
#26 0x7f87dcc9bbac in __clone /home/engshare/third-party2/glibc/2.23/src/glibc-2.23/sysdeps/unix/sysv/linux/x86_64/clone.S:109
SUMMARY: UndefinedBehaviorSanitizer: float-cast-overflow caffe2/aten/src/ATen/native/Normalization.cpp:311:18 in
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment