'thread apply all bt' backtrace of stuck celery worker process

thread apply all.text

(gdb) thread apply all bt


Thread 3 (Thread 0x7f747d028700 (LWP 11546)):
#0  0x00007f74853359d3 in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007f7482c53bed in ?? () from /usr/lib/libpq.so.5
#2  0x00007f7482c53c70 in ?? () from /usr/lib/libpq.so.5
#3  0x00007f7482c51e21 in PQgetResult () from /usr/lib/libpq.so.5
#4  0x00007f7482c520b8 in ?? () from /usr/lib/libpq.so.5
#5  0x00007f7482e7942b in pq_execute_command_locked (conn=0x6428d50, query=
    0x7f7482e9833e "BEGIN", pgres=0x7f747d016338, error=0x7f747d016340, tstate=
    0x7f747d016348) at psycopg/pqpath.c:371
#6  0x00007f7482e79637 in pq_begin_locked (conn=0x6428d50, pgres=
    0x7f747d016338, error=0x7f747d016340, tstate=0x7f747d016348)
    at psycopg/pqpath.c:465
#7  0x00007f7482e7a338 in pq_execute (curs=0x7f7478a53200, query=
    0x7f74788908b4 "SELECT ..."..., async=0, 
    no_result=0, no_begin=0) at psycopg/pqpath.c:919
#8  0x00007f7482e83937 in _psyco_curs_execute (self=0x7f7478a53200, operation=
    'SELECT ... ', vars=
    {...}, async=0, no_result=0) at psycopg/cursor_type.c:444
#9  0x00007f7482e83c79 in psyco_curs_execute (self=0x7f7478a53200, args=
    ('SELECT ...'}), kwargs=0x0)
    at psycopg/cursor_type.c:490
#10 0x00000000004868c4 in PyCFunction_Call (func=
    <built-in method execute of psycopg2._psycopg.cursor object at remote 0x7f7478a53200>, arg=
    ('SELECT ...'}), kw=0x0)
    at ../Objects/methodobject.c:85


Thread 2 (Thread 0x7f7477fff700 (LWP 11557)):
#0  0x00007f748656cfd0 in sem_wait ()
   from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x000000000056d145 in PyThread_acquire_lock (lock=0x7f7470001b20, 
    waitflag=1) at ../Python/thread_pthread.h:321
#2  0x0000000000572267 in lock_PyThread_acquire_lock (self=0x5f15810, args=())
    at ../Modules/threadmodule.c:52
#3  0x00000000004868a6 in PyCFunction_Call (func=
    <built-in method acquire of thread.lock object at remote 0x5f15810>, arg=
    (), kw=0x0) at ../Objects/methodobject.c:81
#4  0x000000000052646a in call_function (pp_stack=0x7f7477ffd240, oparg=0)
    at ../Python/ceval.c:4021
#5  0x0000000000521336 in PyEval_EvalFrameEx (f=
    Frame 0x7f74780a6950, for file /usr/lib/python2.7/threading.py, line 243, in wait (self=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x6381cc0>, acquire=<built-in method acquire of thread.lock object at remote 0x6381cc0>, _Condition__waiters=[<thread.lock at remote 0x5f15810>], release=<built-in method release of thread.lock object at remote 0x6381cc0>) at remote 0x7f74780571b0>, timeout=None, waiter=<thread.lock at remote 0x5f15810>, saved_state=None), throwflag=0) at ../Python/ceval.c:2666

#6  0x0000000000523c3e in PyEval_EvalCodeEx (co=0x23ef880, globals=
    {'current_thread': <function at remote 0x2451c30>, '_BoundedSemaphore': <type at remote 0x2481c00>, 'currentThread': <function at remote 0x2451c30>, '_Timer': <type at remote 0x2483570>, '_format_exc': <function at remote 0x242b648>, 'Semaphore': <function at remote 0x244e648>, 'activeCount': <function at remote 0x2451ed0>, '_profile_hook': None, '_sleep': <built-in function sleep>, '_trace_hook': None, 'ThreadError': <type at remote 0x241b530>, '_enumerate': <function at remote 0x2451f78>, '_start_new_thread': <built-in function start_new_thread>, 'BoundedSemaphore': <function at remote 0x244ed80>, '_shutdown': <instancemethod at remote 0x230c460>, '__all__': ['activeCount', 'active_count', 'Condition', 'currentThread', 'current_thread', 'enumerate', 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Timer', 'setprofile', 'settrace', 'local', 'stack_size'], '_Event': <type at remote 0x2482660>, 'active_count': <function at remote 0x2451ed0>, '__package__': None, '_Condition': <type at remo...(truncated), locals=0x0, args=0x7f7470001398, argcount=1, kws=0x7f74700013a0, 


...

Thread 1 (Thread 0x7f7486992700 (LWP 11464)):
#0  0x00007f748656cfd0 in sem_wait ()
   from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x000000000056d145 in PyThread_acquire_lock (lock=0x432c280, waitflag=1)
---Type <return> to continue, or q <return> to quit---
    at ../Python/thread_pthread.h:321
#2  0x0000000000572267 in lock_PyThread_acquire_lock (self=0x7f74788262c0, 
    args=()) at ../Modules/threadmodule.c:52
#3  0x00000000004868a6 in PyCFunction_Call (func=
    <built-in method acquire of thread.lock object at remote 0x7f74788262c0>, 
    arg=(), kw=0x0) at ../Objects/methodobject.c:81
#4  0x000000000052646a in call_function (pp_stack=0x7fff1e7678f0, oparg=0)
    at ../Python/ceval.c:4021
#5  0x0000000000521336 in PyEval_EvalFrameEx (f=
    Frame 0x6365b40, for file /usr/lib/python2.7/threading.py, line 243, in wait (self=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f7478826180>, acquire=<built-in method acquire of thread.lock object at remote 0x7f7478826180>, _Condition__waiters=[<thread.lock at remote 0x7f74788262c0>], release=<built-in method release of thread.lock object at remote 0x7f7478826180>) at remote 0x67a1bc0>, timeout=None, waiter=<thread.lock at remote 0x7f74788262c0>, saved_state=None), throwflag=0) at ../Python/ceval.c:2666
#6  0x0000000000523c3e in PyEval_EvalCodeEx (co=0x23ef880, globals=
    {'current_thread': <function at remote 0x2451c30>, '_BoundedSemaphore': <type at remote 0x2481c00>, 'currentThread': <function at remote 0x2451c30>, '_Timer': <type at remote 0x2483570>, '_format_exc': <function at remote 0x242b648>, 'Semaphore': <function at remote 0x244e648>, 'activeCount': <function at remote 0x2451ed0>, '_profile_hook': None, '_sleep': <built-in function sleep>, '_trace_hook': None, 'ThreadError': <type at remote 0x241b530>, '_enumerate': <function at remote 0x2451f78>, '_start_new_thread': <built-in function start_new_thread>, 'BoundedSemaphore': <function at remote 0x244ed80>, '_shutdown': <instancemethod at remote 0x230c460>, '__all__': ['activeCount', 'active_count', 'Condition', 'currentThread', 'current_thread', 'enumerate', 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Timer', 'setprofile', 'settrace', 'local', 'stack_size'], '_Event': <type at remote 0x2482660>, 'active_count': <function at remote 0x2451ed0>, '__package__': None, '_Condition': <type at remo...(truncated), locals=0x0, args=0x7f74780e7c68, argcount=1, kws=0x7f74780e7c70, 
    kwcount=0, defs=0x2448e18, defcount=1, closure=0x0)
    at ../Python/ceval.c:3253
#7  0x0000000000526a52 in fast_function (func=<function at remote 0x244eb88>, 
    pp_stack=0x7fff1e767db0, n=1, na=1, nk=0) at ../Python/ceval.c:4117
#8  0x0000000000526641 in call_function (pp_stack=0x7fff1e767db0, oparg=0)
    at ../Python/ceval.c:4042

...