gistfile1.txt

#!/bin/bash
#
# Get a stack trace of the tasks matching the supplied regex
#
# This script was inspired by:
#   http://old.zope.org/Members/4am/debugspinningzope
###############################################################################

#==============================================================================
function Usage
{
    cat <<EOF
Usage: $0 taskregex

Example(s):
    $0 runzope
EOF
    exit 1
}

#==============================================================================
function Die
{
    echo "${*}" >&2
    exit 2
}

#==============================================================================
function DumpPythonTask
{
    local pid="$1"
    local logPrefix="$2"

    local gdbBatch="$logPrefix-$$.gdb"
    local numThreads=$(awk '/^Threads:/ {print $2}' /proc/$pid/status)
    if [ -z "$numThreads" ]; then
        Die "### ERROR: Unable to retrieve numThreads from /proc/$pid/status"
    fi

    case "$numThreads" in
        [0-9]*)
            :  # pass
        ;;

        *)
            Die "### ERROR: numThreads=$numThreads for pid=$pid is not a positive number"
        ;;
    esac

    local tlogPrefix="$logPrefix-$pid-tid"
    local threadId=$numThreads
    local minThreadId=1
    while (( $threadId >= $minThreadId )); do
        log="$tlogPrefix-$threadId.txt"
        echo "thread $threadId" > $gdbBatch
        # call PyRun_SimpleString("import sys, traceback; sys.stderr=open('/tmp/tb','w',0); traceback.print_stack()")
        echo "call PyRun_SimpleString(\"import sys, traceback; sys.stderr=open('$log','w',0); traceback.print_stack()\")" >> $gdbBatch
        let threadId=$(( $threadId - 1 ))
        if gdb --pid $pid --batch --command=$gdbBatch --batch-silent; then
            ( echo -e "\n# stacktrace above is for:"; ps -FLp $pid ) >> $log
            echo "stacktrace for pid:$pid at log: $log"
        fi
    done
}

#==============================================================================
# MAIN

# ---- Check prereqs
which gdb >/dev/null || Die "ERROR: gdb not found - install as root with: yum -y install gdb"
which pgrep >/dev/null || Die "ERROR: pgrep not found - install as root with: yum -y install procps"

# ---- Determine args
if [ $# -lt 1 ]; then
    Usage $0
fi

taskRegex="${1}"

# ---- Get stacktrace of pids of tasks that match supplied regex
mypid=$$
date=$(date +'%Y-%m-%d-%H%M%S')
logPrefix="/tmp/zstack-$date-$(echo $taskRegex | sed -e 's/[^A-Za-z0-9._-]/_/g')"
pids=$(pgrep -f "$taskRegex")
for pid in $pids; do
    if [ "$mypid" = "$pid" ]; then
        continue
    fi
    
    log="$logPrefix-$pid-gstack.txt"
    gstack $pid > $log
    echo "stacktrace for pid:$pid at log: $log"

    DumpPythonTask $pid "$logPrefix"
done

###############################################################################