kineto_hanging.txt

[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
--Type <RET> for more, q to quit, c to continue without paging--
futex_abstimed_wait_cancelable (private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x564019c3bda0)
    at ../sysdeps/nptl/futex-internal.h:320
320     ../sysdeps/nptl/futex-internal.h: No such file or directory.
(gdb) bt
#0  futex_abstimed_wait_cancelable (private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x564019c3bda0)
    at ../sysdeps/nptl/futex-internal.h:320
#1  do_futex_wait (sem=sem@entry=0x564019c3bda0, abstime=0x0, clockid=0) at sem_waitcommon.c:112
#2  0x00007f8a72153548 in __new_sem_wait_slow (sem=0x564019c3bda0, abstime=0x0, clockid=0)
    at sem_waitcommon.c:184
#3  0x00007f89d5c710cf in ?? () from /usr/local/cuda-11.6/extras/CUPTI/lib64/libcupti.so.11.6
#4  0x00007f89d5b13976 in ?? () from /usr/local/cuda-11.6/extras/CUPTI/lib64/libcupti.so.11.6
#5  0x00007f89d5ae2a7b in cuptiActivityFlushAll () from /usr/local/cuda-11.6/extras/CUPTI/lib64/libcupti.so.11.6
#6  0x00007f8a3b27e882 in libkineto::CuptiActivityApi::<lambda()>::operator()(void) const (
    __closure=0x7ffd4c046b68)
    at /scratch/dberard/dynamo38/pytorch/third_party/kineto/libkineto/src/CuptiActivityApi.cpp:167
#7  0x00007f8a3b27ec54 in libkineto::CuptiActivityApi::activityBuffers (
    this=0x7f8a3f166b00 <libkineto::CuptiActivityApi::singleton()::instance>)
    at /scratch/dberard/dynamo38/pytorch/third_party/kineto/libkineto/src/CuptiActivityApi.cpp:167
#8  0x00007f8a3b29673e in libkineto::CuptiActivityProfiler::processTraceInternal (this=0x5640618b5360,
    logger=...)
    at /scratch/dberard/dynamo38/pytorch/third_party/kineto/libkineto/src/CuptiActivityProfiler.cpp:192
#9  0x00007f8a3b2befa7 in libkineto::CuptiActivityProfiler::processTrace (this=0x5640618b5360, logger=...)
    at /scratch/dberard/dynamo38/pytorch/third_party/kineto/libkineto/src/CuptiActivityProfiler.h:147
#10 0x00007f8a3b2be121 in libkineto::ActivityProfilerController::stopTrace (this=0x564019c438c0)
    at /scratch/dberard/dynamo38/pytorch/third_party/kineto/libkineto/src/ActivityProfilerController.cpp:319
#11 0x00007f8a3b291368 in libkineto::ActivityProfilerProxy::stopTrace (this=0x56401b313e10)
    at /scratch/dberard/dynamo38/pytorch/third_party/kineto/libkineto/src/ActivityProfilerProxy.cpp:83
#12 0x00007f8a338c9ea4 in torch::profiler::impl::kineto::stopTrace ()
    at /scratch/dberard/dynamo38/pytorch/torch/csrc/profiler/kineto_shim.cpp:259
#13 0x00007f8a33870585 in torch::profiler::impl::(anonymous namespace)::addKinetoEvents (results=...,
    start_time_us=1679354438535026, end_time_us=1679354438537824, config=...)
    at /scratch/dberard/dynamo38/pytorch/torch/csrc/profiler/collection.cpp:875
#14 0x00007f8a33874f79 in torch::profiler::impl::RecordQueue::getRecords(std::function<long (unsigned long)>, uns
igned long, unsigned long) (this=0x5640618bc1a0, time_converter=..., start_time_us=1679354438535026,
    end_time_us=1679354438537824) at /scratch/dberard/dynamo38/pytorch/torch/csrc/profiler/collection.cpp:1191
#15 0x00007f8a32b17e59 in torch::autograd::profiler::(anonymous namespace)::KinetoThreadLocalState::finalizeTrace
 (this=0x5640618b8280) at /scratch/dberard/dynamo38/pytorch/torch/csrc/autograd/profiler_kineto.cpp:325
#16 0x00007f8a32b19846 in torch::autograd::profiler::disableProfiler ()
    at /scratch/dberard/dynamo38/pytorch/torch/csrc/autograd/profiler_kineto.cpp:690
#17 0x00007f8a40c00593 in pybind11::detail::argument_loader<>::call_impl<std::unique_ptr<torch::autograd::profile
r::ProfilerResult, std::default_delete<torch::autograd::profiler::ProfilerResult> >, std::unique_ptr<torch::autog
rad::profiler::ProfilerResult, std::default_delete<torch::autograd::profiler::ProfilerResult> > (*&)(), , pybind1
1::detail::void_type>(std::unique_ptr<torch::autograd::profiler::ProfilerResult, std::default_delete<torch::autog
rad::profiler::ProfilerResult> > (*&)(), std::integer_sequence<unsigned long>, pybind11::detail::void_type&&) &&
    (this=0x7ffd4c0476e6, f=@0x7f8a43d49348: 0x4474654761647563)
    at /scratch/dberard/dynamo38/pytorch/third_party/pybind11/include/pybind11/cast.h:1439
#18 0x00007ffd4c0476a7 in ?? ()
#19 0x0000563fa03704a8 in ?? ()
#20 0x00007ffd4c0476e6 in ?? ()
#21 0x00007ffd4c0476f0 in ?? ()
#22 0x0000000000000000 in ?? ()
(gdb)