Created
February 23, 2022 23:20
-
-
Save jamesr66a/76aada34569c781cf0d4526ceda43e2d to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #0 0x00007f10250bbd1d in __cxa_throw () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #1 0x00007f101047603d in torch::distributed::rpc::RRefContext::delForkOfOwner(torch::distributed::rpc::GloballyUniqueId const&, torch::distributed::rpc::GloballyUniqueId const&) () from /fsx/users/jamesreed/pytorch/torch/lib/libtorch_cpu.so | |
| #2 0x00007f1010477dc2 in torch::distributed::rpc::RRefContext::notifyOwnerAndParentOfFork(torch::distributed::rpc::GloballyUniqueId const&, short, c10::intrusive_ptr<torch::distributed::rpc::RRef, c10::detail::intrusive_target_default_null_type<torch::distributed::rpc::RRef> > const&) () from /fsx/users/jamesreed/pytorch/torch/lib/libtorch_cpu.so | |
| #3 0x00007f10166da01a in torch::distributed::rpc::PyRRef::unpickle(pybind11::tuple const&) () | |
| from /fsx/users/jamesreed/pytorch/torch/lib/libtorch_python.so | |
| #4 0x00007f10166d1ef7 in void pybind11::cpp_function::initialize<torch::distributed::rpc::PyRRef (*&)(pybind11::tuple const&), torch::distributed::rpc::PyRRef, pybind11::tuple const&, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::call_guard<pybind11::gil_scoped_release> >(torch::distributed::rpc::PyRRef (*&)(pybind11::tuple const&), torch::distributed::rpc::PyRRef (*)(pybind11::tuple const&), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&, pybind11::call_guard<pybind11::gil_scoped_release> const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call&) () | |
| from /fsx/users/jamesreed/pytorch/torch/lib/libtorch_python.so | |
| #5 0x00007f101603c53a in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () | |
| from /fsx/users/jamesreed/pytorch/torch/lib/libtorch_python.so | |
| #6 0x0000558250979e64 in cfunction_call () | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/methodobject.c:539 | |
| #7 0x0000558250973e4f in _PyObject_MakeTpCall () | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/call.c:191 | |
| #8 0x0000558250a03e29 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x7f0fb85fff48, | |
| callable=<optimized out>, tstate=<optimized out>) | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Include/cpython/abstract.h:116 | |
| #9 PyObject_Vectorcall () at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Include/cpython/abstract.h:127 | |
| #10 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7f0f1000ad70) | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Python/ceval.c:5072 | |
| #11 _PyEval_EvalFrameDefault () at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Python/ceval.c:3487 | |
| #12 0x00005582509ba8c3 in _PyEval_EvalFrame () | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Include/internal/pycore_ceval.h:40 | |
| #13 function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>, | |
| tstate=0x7f0f1000ad70) at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/call.c:330 | |
| #14 _PyFunction_Vectorcall () at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/call.c:367 | |
| #15 0x00005582509e4c6b in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=2, args=0x7f0fa07f7030, callable=0x7f0fbc5da280, | |
| tstate=0x7f0f1000ad70) at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Include/cpython/abstract.h:118 | |
| #16 method_vectorcall () at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/classobject.c:83 | |
| #17 0x0000558250973788 in PyVectorcall_Call (kwargs=<optimized out>, tuple=0x7f0fbb715ac0, callable=0x7f0fb85f9780) | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/call.c:231 | |
| #18 _PyObject_Call () at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/call.c:266 | |
| #19 0x00007f10253db0f0 in load_reduce.isra.36 () at /usr/local/src/conda/python-3.9.5/Modules/_pickle.c:6833 | |
| #20 load () at /usr/local/src/conda/python-3.9.5/Modules/_pickle.c:6981 | |
| #21 0x00005582509b3813 in method_vectorcall_NOARGS () | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/descrobject.c:434 | |
| #22 0x00005582509321a4 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x7f0f1000c620, | |
| callable=0x7f0fc296cc70, tstate=<optimized out>) | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Include/cpython/abstract.h:118 | |
| #23 PyObject_Vectorcall () at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Include/cpython/abstract.h:127 | |
| #24 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7f0f1000ad70) | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Python/ceval.c:5072 | |
| #25 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=0x7f0f1000c470, throwflag=<optimized out>) | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Python/ceval.c:3504 | |
| #26 0x00005582509ba8c3 in _PyEval_EvalFrame () | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Include/internal/pycore_ceval.h:40 | |
| #27 function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>, | |
| tstate=0x7f0f1000ad70) at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/call.c:330 | |
| #28 _PyFunction_Vectorcall () at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/call.c:367 | |
| #29 0x00005582509321a4 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x7f0fb86076d0, | |
| callable=0x7f0fbc5da5e0, tstate=<optimized out>) | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Include/cpython/abstract.h:118 | |
| #30 PyObject_Vectorcall () at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Include/cpython/abstract.h:127 | |
| #31 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7f0f1000ad70) | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Python/ceval.c:5072 | |
| #32 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=0x7f0fb8607550, throwflag=<optimized out>) | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Python/ceval.c:3504 | |
| #33 0x00005582509ba8c3 in _PyEval_EvalFrame () | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Include/internal/pycore_ceval.h:40 | |
| #34 function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>, | |
| tstate=0x7f0f1000ad70) at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/call.c:330 | |
| #35 _PyFunction_Vectorcall () at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/call.c:367 | |
| #36 0x0000558250973788 in PyVectorcall_Call (kwargs=<optimized out>, tuple=0x7f0fb862e480, callable=0x7f0fbc5da670) | |
| at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/call.c:231 | |
| #37 _PyObject_Call () at /home/builder/ktietz/cos6/ci_cos6/python-split_1622827272909/work/Objects/call.c:266 | |
| #38 0x00007f10166e8776 in torch::distributed::rpc::PythonRpcHandler::deserialize(torch::distributed::rpc::SerializedPyObj const&) | |
| () from /fsx/users/jamesreed/pytorch/torch/lib/libtorch_python.so | |
| #39 0x00007f10166fb2ae in torch::distributed::rpc::UnpickledPythonCall::UnpickledPythonCall(torch::distributed::rpc::SerializedPyObj const&, bool) () from /fsx/users/jamesreed/pytorch/torch/lib/libtorch_python.so | |
| #40 0x00007f10166ea4e4 in torch::distributed::rpc::(anonymous namespace)::deserializePythonRpcCommandReference(torch::distributed::rpc::RpcCommandBase&, torch::distributed::rpc::MessageType const&) [clone .isra.796] () | |
| from /fsx/users/jamesreed/pytorch/torch/lib/libtorch_python.so | |
| #41 0x00007f10166ea5be in torch::distributed::rpc::RequestCallbackImpl::deserializePythonRpcCommand(std::unique_ptr<torch::distributed::rpc::RpcCommandBase, std::default_delete<torch::distributed::rpc::RpcCommandBase> >, torch::distributed::rpc::MessageType const&) const () from /fsx/users/jamesreed/pytorch/torch/lib/libtorch_python.so | |
| #42 0x00007f10104632f6 in torch::distributed::rpc::RequestCallbackNoPython::processMessage(torch::distributed::rpc::Message&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const () from /fsx/users/jamesreed/pytorch/torch/lib/libtorch_cpu.so | |
| #43 0x00007f101045e067 in torch::distributed::rpc::RequestCallback::operator()(torch::distributed::rpc::Message&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const () from /fsx/users/jamesreed/pytorch/torch/lib/libtorch_cpu.so |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment