occupancy_hang.txt

Thread 9 (Thread 0x7f2da4fe8700 (LWP 39329)):
Undefined command: "btr".  Try "help".

Thread 9 (Thread 0x7f2da4fe8700 (LWP 39329)):
#0  0x00007f2dccf7b709 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x000055787b3902d8 in PyCOND_TIMEDWAIT (cond=0x55787b5c2a38 <_PyRuntime+1208>, mut=0x55787b5c2a68 <_PyRuntime+1256>, 
    us=5000) at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/condvar.h:90
#2  take_gil (tstate=0x55787f257a40)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval_gil.h:208
#3  PyEval_RestoreThread () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:271
#4  0x000055787b45f970 in PyGILState_Ensure ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/pystate.c:1067
#5  0x00007f2dcafe79d7 in _CallPythonObject (pArgs=0x7f2da4fe5480, flags=4353, converters=0x7f2dc08b60b8, 
    callable=0x7f28841d3e18, setfunc=0x7f2dcafe29b0 <L_set>, restype=0x7f2dcb01c4c8, mem=0x7f2da4fe5610)
    at /usr/local/src/conda/python-3.7.3/Modules/_ctypes/callbacks.c:140
#6  closure_fcn () at /usr/local/src/conda/python-3.7.3/Modules/_ctypes/callbacks.c:292
#7  0x00007f2dcafd13d0 in ffi_closure_unix64_inner ()
   from /home/nfs/pentschev/miniconda3/envs/ucx/lib/python3.7/lib-dynload/../../libffi.so.6
#8  0x00007f2dcafd1798 in ffi_closure_unix64 ()
   from /home/nfs/pentschev/miniconda3/envs/ucx/lib/python3.7/lib-dynload/../../libffi.so.6
#9  0x00007f2dc90c9297 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#10 0x00007f2dc8fe5d67 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#11 0x00007f2dc8fe5d90 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#12 0x00007f2dc911e3b6 in cuOccupancyMaxPotentialBlockSize () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#13 0x00007f2dcafd1630 in ffi_call_unix64 ()
   from /home/nfs/pentschev/miniconda3/envs/ucx/lib/python3.7/lib-dynload/../../libffi.so.6
#14 0x00007f2dcafd0fed in ffi_call () from /home/nfs/pentschev/miniconda3/envs/ucx/lib/python3.7/lib-dynload/../../libffi.so.6
#15 0x00007f2dcafe7fce in _call_function_pointer (argcount=6, resmem=0x7f2da4fe5a70, restype=<optimized out>, 
    atypes=0x7f2da4fe59f0, avalues=0x7f2da4fe5a30, pProc=0x7f2dc911e330 <cuOccupancyMaxPotentialBlockSize>, flags=4353)
    at /usr/local/src/conda/python-3.7.3/Modules/_ctypes/callproc.c:827
#16 _ctypes_callproc () at /usr/local/src/conda/python-3.7.3/Modules/_ctypes/callproc.c:1184
#17 0x00007f2dcafe8a04 in PyCFuncPtr_call () at /usr/local/src/conda/python-3.7.3/Modules/_ctypes/_ctypes.c:3969
#18 0x000055787b39c51e in PyObject_Call ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:245
#19 0x000055787b44603c in do_call_core (kwdict=0x0, callargs=0x7f28600c74c8, func=0x7f28841fe048)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#20 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#21 0x000055787b38a1d9 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#22 0x000055787b3daf87 in _PyFunction_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:433
#23 0x000055787b448c59 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#24 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3093
#25 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#26 0x000055787b38ac20 in _PyFunction_FastCallDict ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:376
#27 0x000055787b3a9e23 in _PyObject_Call_Prepend ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:904
#28 0x000055787b39c51e in PyObject_Call ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:245
#29 0x000055787b44603c in do_call_core (kwdict=0x7f28608e9c60, callargs=0x7f2dcd357048, func=0x7f286019adc8)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#30 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#31 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=2, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#32 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#33 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#34 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#35 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#36 0x000055787b38a9f5 in _PyFunction_FastCallDict ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:376
#37 0x000055787b3a9e23 in _PyObject_Call_Prepend ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:904
#38 0x000055787b3e32ea in slot_tp_call ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/typeobject.c:6379
#39 0x000055787b3e416b in _PyObject_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:199
#40 0x000055787b448bb6 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4619
#41 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3124
#42 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#43 0x000055787b3daf87 in _PyFunction_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:433
#44 0x000055787b4455cc in call_function (kwnames=0x7f2884c60c18, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#45 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3139
#46 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#47 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#48 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#49 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#50 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#51 0x000055787b3daf87 in _PyFunction_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:433
#52 0x000055787b444506 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#53 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3124
#54 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=2, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#55 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#56 0x000055787b448c59 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#57 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3093
#58 0x000055787b38a1d9 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#59 0x000055787b3daf87 in _PyFunction_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:433
#60 0x000055787b4455cc in call_function (kwnames=0x7f2884bd3a08, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#61 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3139
#62 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#63 0x000055787b38a9f5 in _PyFunction_FastCallDict ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:376
#64 0x000055787b44603c in do_call_core (kwdict=0x7f286c34cd80, callargs=0x7f286c286fd0, func=0x7f2884bd52f0)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#65 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#66 0x000055787b38a92b in function_code_fastcall (globals=<optimized out>, nargs=8, args=<optimized out>, co=0x7f2dc87aa150)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#67 _PyFunction_FastCallDict () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:322
#68 0x000055787b44603c in do_call_core (kwdict=0x7f2884052168, callargs=0x7f28840f5d68, func=0x7f2dc86f5598)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#69 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#70 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#71 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#72 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#73 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#74 0x000055787b38a92b in function_code_fastcall (globals=<optimized out>, nargs=2, args=<optimized out>, co=0x7f2dca8c80c0)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#75 _PyFunction_FastCallDict () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:322
#76 0x000055787b44603c in do_call_core (kwdict=0x7f2da0a5b288, callargs=0x7f2da5e896c8, func=0x7f2dca8cd1e0)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#77 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#78 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#79 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#80 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#81 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#82 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#83 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#84 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#85 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#86 0x000055787b38a92b in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=0x7f2dcba6e1e0)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#87 _PyFunction_FastCallDict () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:322
#88 0x000055787b3a9e23 in _PyObject_Call_Prepend ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:904
#89 0x000055787b39c51e in PyObject_Call ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:245
#90 0x000055787b49b9e7 in t_bootstrap ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/_threadmodule.c:994
#91 0x000055787b456fe8 in pythread_wrapper (arg=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/thread_pthread.h:174
#92 0x00007f2dccf756ba in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#93 0x00007f2dcccab41d in clone () from /lib/x86_64-linux-gnu/libc.so.6

Thread 8 (Thread 0x7f2da5dec700 (LWP 39318)):
#0  0x00007f2dccf7d827 in do_futex_wait.constprop () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00007f2dccf7d8d4 in __new_sem_wait_slow.constprop.0 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007f2dccf7d97a in sem_wait@@GLIBC_2.2.5 () from /lib/x86_64-linux-gnu/libpthread.so.0
#3  0x000055787b38f409 in PyThread_acquire_lock_timed ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/thread_pthread.h:372
#4  0x00007f2dcb480250 in _queue_SimpleQueue_get_impl (timeout=<optimized out>, block=<optimized out>, self=0x7f2dcaafc908)
    at /usr/local/src/conda/python-3.7.3/Modules/_queuemodule.c:222
#5  _queue_SimpleQueue_get (self=0x7f2dcaafc908, args=<optimized out>, nargs=<optimized out>, kwnames=<optimized out>)
    at /usr/local/src/conda/python-3.7.3/Modules/clinic/_queuemodule.c.h:136
#6  0x000055787b3db72c in _PyMethodDef_RawFastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:656
#7  0x000055787b3db891 in _PyCFunction_FastCallKeywords (func=0x7f28603ec900, args=<optimized out>, nargs=<optimized out>, 
    kwnames=<optimized out>) at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:730
#8  0x000055787b449574 in call_function (kwnames=0x7f2dcac4b1d0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4568
#9  _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3139
#10 0x000055787b38a92b in function_code_fastcall (globals=<optimized out>, nargs=4, args=<optimized out>, co=0x7f2dcac4a9c0)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#11 _PyFunction_FastCallDict () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:322
#12 0x000055787b44603c in do_call_core (kwdict=0x7f2da5eb0bd0, callargs=0x7f2da5e8d278, func=0x7f2dcac2e7b8)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#13 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#14 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#15 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#16 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#17 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#18 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#19 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#20 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#21 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#22 0x000055787b38a92b in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=0x7f2dcba6e1e0)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#23 _PyFunction_FastCallDict () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:322
#24 0x000055787b3a9e23 in _PyObject_Call_Prepend ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:904
#25 0x000055787b39c51e in PyObject_Call ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:245
#26 0x000055787b49b9e7 in t_bootstrap ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/_threadmodule.c:994
#27 0x000055787b456fe8 in pythread_wrapper (arg=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/thread_pthread.h:174
#28 0x00007f2dccf756ba in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#29 0x00007f2dcccab41d in clone () from /lib/x86_64-linux-gnu/libc.so.6

Thread 7 (Thread 0x7f2da6754700 (LWP 39244)):
#0  0x00007f2dccc9f74d in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007f2dc90e5633 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#2  0x00007f2dc914e84d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#3  0x00007f2dc90e76a8 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#4  0x00007f2dccf756ba in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5  0x00007f2dcccab41d in clone () from /lib/x86_64-linux-gnu/libc.so.6

Thread 6 (Thread 0x7f2da8138700 (LWP 39236)):
#0  0x00007f2dcccaba13 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007f2dc9ea132b in ucs_event_set_wait (event_set=0x55787d2232d0, num_events=0x7f2da8137ee8, timeout_ms=100, 
    event_set_handler=0x7f2dc9e89453 <ucs_async_thread_ev_handler>, arg=0x7f2da8137f30) at ../../../src/ucs/sys/event_set.c:196
#2  0x00007f2dc9e89696 in ucs_async_thread_func (arg=0x55787ce82ac0) at ../../../src/ucs/async/thread.c:127
#3  0x00007f2dccf756ba in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#4  0x00007f2dcccab41d in clone () from /lib/x86_64-linux-gnu/libc.so.6

Thread 5 (Thread 0x7f2da975c700 (LWP 39230)):
#0  0x00007f2dcccac8c8 in accept4 () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007f2dc90e657a in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#2  0x00007f2dc90d8abd in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#3  0x00007f2dc90e76a8 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#4  0x00007f2dccf756ba in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5  0x00007f2dcccab41d in clone () from /lib/x86_64-linux-gnu/libc.so.6

Thread 4 (Thread 0x7f2dbd029700 (LWP 39220)):
#0  0x00007f2dccc9f74d in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007f2dcb8e02a4 in poll_poll () at /usr/local/src/conda/python-3.7.3/Modules/selectmodule.c:590
#2  0x000055787b3db6f0 in _PyMethodDef_RawFastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:694
#3  0x000055787b3e3cdf in _PyMethodDescr_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/descrobject.c:288
#4  0x000055787b448dac in call_function (kwnames=0x0, oparg=2, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4593
#5  _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#6  0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#7  0x000055787b3daf25 in _PyFunction_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:433
#8  0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#9  _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#10 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#11 0x000055787b3daf25 in _PyFunction_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:433
#12 0x000055787b444506 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#13 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3124
#14 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=2, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#15 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#16 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#17 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#18 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#19 0x000055787b3daf25 in _PyFunction_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:433
#20 0x000055787b448c59 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#21 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3093
#22 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#23 0x000055787b3daf87 in _PyFunction_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:433
#24 0x000055787b4455cc in call_function (kwnames=0x7f2dc85fc048, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#25 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3139
#26 0x000055787b38a1d9 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#27 0x000055787b38a9f5 in _PyFunction_FastCallDict ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:376
#28 0x000055787b44603c in do_call_core (kwdict=0x7f2dbd031c18, callargs=0x7f2dcd357048, func=0x7f2dbd035488)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#29 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#30 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#31 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#32 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#33 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#34 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#35 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#36 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#37 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#38 0x000055787b38a92b in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=0x7f2dcba6e1e0)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#39 _PyFunction_FastCallDict () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:322
#40 0x000055787b3a9e23 in _PyObject_Call_Prepend ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:904
#41 0x000055787b39c51e in PyObject_Call ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:245
#42 0x000055787b49b9e7 in t_bootstrap ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/_threadmodule.c:994
#43 0x000055787b456fe8 in pythread_wrapper (arg=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/thread_pthread.h:174
#44 0x00007f2dccf756ba in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#45 0x00007f2dcccab41d in clone () from /lib/x86_64-linux-gnu/libc.so.6

Thread 3 (Thread 0x7f2dbd86a700 (LWP 39219)):
#0  0x00007f2dccf7b709 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x000055787b3902d8 in PyCOND_TIMEDWAIT (cond=0x55787b5c2a38 <_PyRuntime+1208>, mut=0x55787b5c2a68 <_PyRuntime+1256>, 
    us=5000) at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/condvar.h:90
#2  take_gil (tstate=0x55787ccdfc90)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval_gil.h:208
#3  PyEval_RestoreThread () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:271
#4  0x000055787b3ffa50 in pysleep (secs=10000000)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/timemodule.c:1830
#5  time_sleep () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/timemodule.c:371
#6  0x000055787b3db59c in _PyMethodDef_RawFastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:644
#7  0x000055787b3db891 in _PyCFunction_FastCallKeywords (func=0x7f2dcba771b0, args=<optimized out>, nargs=<optimized out>, 
    kwnames=<optimized out>) at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:730
#8  0x000055787b4488c4 in call_function (kwnames=0x0, oparg=1, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4568
#9  _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3124
#10 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#11 0x000055787b38ac20 in _PyFunction_FastCallDict ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:376
#12 0x000055787b44603c in do_call_core (kwdict=0x7f2dbd89e870, callargs=0x7f2dcd357048, func=0x7f2dc87c2d08)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#13 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#14 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#15 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#16 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#17 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#18 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#19 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#20 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#21 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#22 0x000055787b38a92b in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=0x7f2dcba6e1e0)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#23 _PyFunction_FastCallDict () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:322
#24 0x000055787b3a9e23 in _PyObject_Call_Prepend ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:904
#25 0x000055787b39c51e in PyObject_Call ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:245
#26 0x000055787b49b9e7 in t_bootstrap ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/_threadmodule.c:994
#27 0x000055787b456fe8 in pythread_wrapper (arg=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/thread_pthread.h:174
#28 0x00007f2dccf756ba in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#29 0x00007f2dcccab41d in clone () from /lib/x86_64-linux-gnu/libc.so.6

Thread 2 (Thread 0x7f2dc07bb700 (LWP 39211)):
#0  0x00007f2dccf7e51d in read () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x000055787b3fbf57 in read ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/_build_env/x86_64-conda_cos6-linux-gnu/sysroot/usr/include/bits/unistd.h:45
#2  _Py_read () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/fileutils.c:1407
#3  0x000055787b3fc095 in os_read_impl.isra.68 (length=4, fd=29)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/posixmodule.c:8043
#4  os_read () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/clinic/posixmodule.c.h:3625
#5  0x000055787b3db5f9 in _PyMethodDef_RawFastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:651
#6  0x000055787b3db891 in _PyCFunction_FastCallKeywords (func=0x7f2dcd30b0d8, args=<optimized out>, nargs=<optimized out>, 
    kwnames=<optimized out>) at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:730
#7  0x000055787b4488c4 in call_function (kwnames=0x0, oparg=2, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4568
#8  _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3124
#9  0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#10 0x000055787b3daf25 in _PyFunction_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:433
#11 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#12 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#13 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#14 0x000055787b3daf25 in _PyFunction_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:433
#15 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#16 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#17 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#18 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#19 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#20 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#21 0x000055787b389c0a in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#22 0x000055787b38a9f5 in _PyFunction_FastCallDict ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:376
#23 0x000055787b44603c in do_call_core (kwdict=0x7f2dc18f4828, callargs=0x7f2dcd357048, func=0x7f2dc82d70d0)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#24 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#25 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#26 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#27 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#28 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#29 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#30 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#31 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#32 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#33 0x000055787b38a92b in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=0x7f2dcba6e1e0)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#34 _PyFunction_FastCallDict () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:322
#35 0x000055787b3a9e23 in _PyObject_Call_Prepend ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:904
#36 0x000055787b39c51e in PyObject_Call ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:245
#37 0x000055787b49b9e7 in t_bootstrap ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/_threadmodule.c:994
#38 0x000055787b456fe8 in pythread_wrapper (arg=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/thread_pthread.h:174
#39 0x00007f2dccf756ba in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#40 0x00007f2dcccab41d in clone () from /lib/x86_64-linux-gnu/libc.so.6

Thread 1 (Thread 0x7f2dcd398700 (LWP 39197)):
#0  0x00007f2dccf7e26d in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00007f2dccf77e42 in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007f2dc90c6bf6 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#3  0x00007f2dc90c6c28 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#4  0x00007f2dc8fdb1b1 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#5  0x00007f2dc8fddfe6 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#6  0x00007f2dc912ae02 in cuMemcpyDtoHAsync_v2 () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#7  0x00007f2dc8cab70f in ?? () from /home/nfs/pentschev/miniconda3/envs/ucx/lib/libcudart.so.9.2
#8  0x00007f2dc8c8b173 in ?? () from /home/nfs/pentschev/miniconda3/envs/ucx/lib/libcudart.so.9.2
#9  0x00007f2dc8cc1348 in cudaMemcpyAsync () from /home/nfs/pentschev/miniconda3/envs/ucx/lib/libcudart.so.9.2
#10 0x00007f2dbc6205b7 in uct_cuda_copy_ep_get_short (tl_ep=0x55787d2038c0, buffer=0x7f28fc40ee0a, length=389448, 
    remote_addr=139808776984784, rkey=18446744073709551615) at ../../../../src/uct/cuda/cuda_copy/cuda_copy_ep.c:159
#11 0x00007f2dca54a9cb in uct_ep_get_short (rkey=18446744073709551615, remote_addr=139808776984784, length=389448, 
    buffer=0x7f28fc40ee0a, ep=0x55787d2038c0) at /home/nfs/pentschev/ucx-src/ucx/build/../src/uct/api/uct.h:2343
#12 ucp_mem_type_pack (worker=0x55787d59b830, dest=0x7f28fc40ee0a, src=0x7f27c47d18d0, length=389448, 
    mem_type=UCS_MEMORY_TYPE_CUDA) at ../../../src/ucp/dt/dt.c:85
#13 0x00007f2dca54ab37 in ucp_dt_pack (worker=0x55787d59b830, datatype=8, mem_type=UCS_MEMORY_TYPE_CUDA, dest=0x7f28fc40ee0a, 
    src=0x7f27c47d18d0, state=0x55787e374360, length=389448) at ../../../src/ucp/dt/dt.c:114
#14 0x00007f2dca567577 in ucp_tag_pack_eager_only_dt (dest=0x7f28fc40ee02, arg=0x55787e374300)
    at ../../../src/ucp/tag/eager_snd.c:30
#15 0x00007f2dbc37f86f in uct_rc_mlx5_ep_am_bcopy (tl_ep=0x55787f3422f0, id=2 '\002', 
    pack_cb=0x7f2dca5674bd <ucp_tag_pack_eager_only_dt>, arg=0x55787e374300, flags=0)
    at ../../../../src/uct/ib/rc/accel/rc_mlx5_ep.c:315
#16 0x00007f2dca567f31 in uct_ep_am_bcopy (flags=0, arg=0x55787e374300, pack_cb=0x7f2dca5674bd <ucp_tag_pack_eager_only_dt>, 
    id=2 '\002', ep=0x55787f3422f0) at /home/nfs/pentschev/ucx-src/ucx/build/../src/uct/api/uct.h:2418
#17 ucp_do_am_bcopy_single (pack_cb=0x7f2dca5674bd <ucp_tag_pack_eager_only_dt>, am_id=2 '\002', self=0x55787e3743a8)
    at /home/nfs/pentschev/ucx-src/ucx/build/../src/ucp/proto/proto_am.inl:28
#18 ucp_tag_eager_bcopy_single (self=0x55787e3743a8) at ../../../src/ucp/tag/eager_snd.c:136
#19 0x00007f2dca586768 in ucp_request_try_send (pending_flags=0, req_status=0x7ffd84eb4290, req=0x55787e374300)
    at /home/nfs/pentschev/ucx-src/ucx/build/../src/ucp/core/ucp_request.inl:171
#20 ucp_request_send (pending_flags=0, req=0x55787e374300)
    at /home/nfs/pentschev/ucx-src/ucx/build/../src/ucp/core/ucp_request.inl:206
#21 ucp_tag_send_req (enable_zcopy=1, proto=0x7f2dca7ca9a0 <ucp_tag_eager_proto>, cb=0x7f2dca8942d0 <send_handle>, 
    rndv_am_thresh=18446744073709551615, rndv_rma_thresh=2097150, msg_config=0x55787d59d428, dt_count=389448, 
    req=0x55787e374300) at ../../../src/ucp/tag/tag_send.c:108
#22 ucp_tag_send_nb (ep=0x7f2da67d50d8, buffer=0x7f27c47d18d0, count=389448, datatype=8, tag=4113749268370025533, 
    cb=0x7f2dca8942d0 <send_handle>) at ../../../src/ucp/tag/tag_send.c:223
#23 0x00007f2dca8948b3 in ucp_py_ep_send_nb (internal_ep=0x55787e3b5e10, send_buf=0x7f27c47d18d0, length=389448)
    at ucp/_libs/src/ucp_py_ucp_fxns.c:373
#24 0x00007f2dca82a682 in __pyx_pf_6ucp_py_8Endpoint_22send_obj (__pyx_v_name=<optimized out>, __pyx_v_nbytes=0x7f285b248f50, 
    __pyx_v_msg=<optimized out>, __pyx_v_self=0x7f2da5f0b8b0) at ucp/_libs/ucp_py.c:22691
#25 __pyx_pw_6ucp_py_8Endpoint_23send_obj (__pyx_v_self=0x7f2da5f0b8b0, __pyx_args=<optimized out>, __pyx_kwds=<optimized out>)
    at ucp/_libs/ucp_py.c:22426
#26 0x000055787b3db774 in _PyMethodDef_RawFastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:690
#27 0x000055787b3e3cdf in _PyMethodDescr_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/descrobject.c:288
#28 0x000055787b448dac in call_function (kwnames=0x0, oparg=2, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4593
#29 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#30 0x000055787b3e4d4c in gen_send_ex (closing=0, exc=0, arg=<optimized out>, gen=0x7f28601e5348)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/genobject.c:221
#31 _PyGen_Send () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/genobject.c:291
#32 0x000055787b445bbc in _PyEval_EvalFrameDefault ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:1787
#33 0x000055787b3e4d4c in gen_send_ex (closing=0, exc=0, arg=<optimized out>, gen=0x7f28601e5848)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/genobject.c:221
#34 _PyGen_Send () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/genobject.c:291
#35 0x00007f2dcae37bc7 in task_step_impl (exc=<optimized out>, task=0x7f28602cccc8)
    at /usr/local/src/conda/python-3.7.3/Modules/_asynciomodule.c:2558
#36 task_step (exc=0x0, task=0x7f28602cccc8) at /usr/local/src/conda/python-3.7.3/Modules/_asynciomodule.c:2871
#37 task_wakeup (o=<optimized out>, task=0x7f28602cccc8) at /usr/local/src/conda/python-3.7.3/Modules/_asynciomodule.c:2908
#38 TaskWakeupMethWrapper_call () at /usr/local/src/conda/python-3.7.3/Modules/_asynciomodule.c:1797
#39 0x000055787b3e416b in _PyObject_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:199
#40 0x000055787b477da3 in context_run ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/context.c:618
#41 0x000055787b3abcd9 in _PyMethodDef_RawFastCallDict ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:544
#42 0x000055787b3abf01 in _PyCFunction_FastCallDict (func=0x7f28602548b8, args=<optimized out>, nargs=<optimized out>, 
    kwargs=<optimized out>) at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:582
#43 0x000055787b449d54 in do_call_core (kwdict=0x0, callargs=0x7f2860315c48, func=0x7f28602548b8)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4641
#44 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#45 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#46 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#47 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#48 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#49 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#50 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#51 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#52 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#53 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#54 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#55 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#56 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#57 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#58 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#59 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#60 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#61 0x000055787b38a1d9 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#62 0x000055787b3daf87 in _PyFunction_FastCallKeywords ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:433
#63 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#64 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#65 0x000055787b38a1d9 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#66 0x000055787b38ac20 in _PyFunction_FastCallDict ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:376
#67 0x000055787b3a9e23 in _PyObject_Call_Prepend ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:904
#68 0x000055787b39c51e in PyObject_Call ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:245
#69 0x000055787b44603c in do_call_core (kwdict=0x7f2dc82d9510, callargs=0x7f2dcd357048, func=0x7f2dcb423ac8)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#70 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#71 0x000055787b38a92b in function_code_fastcall (globals=<optimized out>, nargs=6, args=<optimized out>, co=0x7f2dc8586390)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#72 _PyFunction_FastCallDict () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:322
#73 0x000055787b3a9e9e in _PyObject_Call_Prepend ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:904
#74 0x000055787b39c51e in PyObject_Call ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:245
#75 0x000055787b44603c in do_call_core (kwdict=0x7f2dc07c3c60, callargs=0x7f2dc7e4cd00, func=0x7f2dcd349cc8)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#76 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#77 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#78 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#79 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#80 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#81 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#82 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#83 0x000055787b4447e0 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#84 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3110
#85 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#86 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#87 0x000055787b448c59 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#88 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3093
#89 0x000055787b3dacfb in function_code_fastcall (globals=<optimized out>, nargs=4, args=<optimized out>, co=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:283
#90 _PyFunction_FastCallKeywords () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:408
#91 0x000055787b444506 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4616
#92 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3124
#93 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#94 0x000055787b38ac20 in _PyFunction_FastCallDict ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Objects/call.c:376
#95 0x000055787b44603c in do_call_core (kwdict=0x7f2dcd34df78, callargs=0x7f2dcb911990, func=0x7f2dcd1d9510)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:4645
#96 _PyEval_EvalFrameDefault () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3191
#97 0x000055787b389929 in _PyEval_EvalCodeWithName ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3930
#98 0x000055787b38a7e4 in PyEval_EvalCodeEx ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:3959
#99 0x000055787b38a80c in PyEval_EvalCode (co=<optimized out>, globals=<optimized out>, locals=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/ceval.c:524
#100 0x000055787b4a2ac4 in run_mod ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/pythonrun.c:1035
#101 0x000055787b4add0d in PyRun_StringFlags ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/pythonrun.c:959
#102 0x000055787b4add6f in PyRun_SimpleStringFlags ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Python/pythonrun.c:455
#103 0x000055787b4ade6d in pymain_run_command (cf=0x7ffd84eb6490, command=<optimized out>)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/main.c:385
#104 pymain_run_python (pymain=0x7ffd84eb65a0)
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/main.c:2871
#105 pymain_main () at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/main.c:3038
#106 0x000055787b4ae1dc in _Py_UnixMain ()
    at /home/conda/feedstock_root/build_artifacts/python_1562015400360/work/Modules/main.c:3073
#107 0x00007f2dccbc4830 in __libc_start_main () from /lib/x86_64-linux-gnu/libc.so.6
#108 0x000055787b453d3d in _start () at ../sysdeps/x86_64/elf/start.S:103