Quick dump htop+top+threadDump for analysis to identify CPU top consumer in a Java process (ex: Tomcat)

dumpCPUStack.sh

#!/bin/bash

COUNTER=2
TARGETDIR=/home/jalios/dumpCPUStack
PID=$(cat /home/jalios/instance/tomcat.pid)
#PID=$(ps h -fu tomcat | grep jplatform | awk '{print $2}')

echo "This script will dump $COUNTER times the threads stacks and CPU usage for process pid=$PID"

while [ $COUNTER -gt 0 ]; do
  TIMESTAMP=$(date +%Y%m%d%H%M%S)

  OUTFILE_STAT="$TARGETDIR"/thread_cpu_"$TIMESTAMP".txt
  echo "Dumping threads statistics : ${OUTFILE_STAT}"
  {
    echo "Thread CPU stats for process $PID"
    echo -n " Memory: "
    ps --pid "$PID" -o rss |grep -E "^\s*[0-9]+" | xargs -i% echo "% / 1024" | bc | xargs printf "%s MB\n"
    echo -n " Threads: "
    ps --pid "$PID" -o nlwp |grep -E "^\s*[0-9]+"
    echo -e "TID-H\tTID\t%CPU\tCPUTIME"
    ps --pid "$PID" -L -o tid,tid,pcpu,cputime |grep -E "^\s*[0-9]+" |sort -rn -k 2 |xargs -n 4 printf '0x%x\t%s\t%s\t%s\n'
  } >>"${OUTFILE_STAT}"

  OUTFILE_SAMPLE="$TARGETDIR"/thread_sample_"$TIMESTAMP".txt
  echo "Dumping threads usage sample : ${OUTFILE_SAMPLE}"
  top -H -p "$PID" -n1 -b > "${OUTFILE_SAMPLE}"

  OUTFILE_TDUMP="$TARGETDIR"/threaddump_"$TIMESTAMP".txt
  echo "Dumping theads (java) : ${OUTFILE_TDUMP}"
  jstack -l "$PID" > "${OUTFILE_TDUMP}"
  #sudo su -l tomcat -s /bin/bash -c 'jstack -l "$PID"' > "${OUTFILE_TDUMP}"

  OUTFILE_REPORT="$TARGETDIR"/report_"$TIMESTAMP".txt
  FIRST_TID=$(head -n 8 "${OUTFILE_SAMPLE}" | tail -n 1 | awk '{print $1}')
  FIRST_TID_CPU=$(head -n 8 "${OUTFILE_SAMPLE}" | tail -n 1 | awk '{print int($9)}')
  if ((FIRST_TID_CPU > 5))
  then
    FIRST_TID_HEX=$(grep -P "^0x\w+\s+${FIRST_TID}\s+" ${OUTFILE_STAT} | awk '{print $1}')
    echo "Writing report : ${OUTFILE_REPORT}"
    {
      echo "Source files for this report:"
      echo " - ${OUTFILE_STAT}"
      echo " - ${OUTFILE_SAMPLE}"
      echo " - ${OUTFILE_TDUMP}"
      echo "*** First thread in sample (${FIRST_TID} - ${FIRST_TID_HEX})"
      head -n 8 "${OUTFILE_SAMPLE}" | tail -n 2
      echo "*** Gobal consumption"
      grep -P "^${FIRST_TID_HEX}\s+" ${OUTFILE_STAT}  | awk '{print "%CPU: "$3" - CPUTIME: "$4}'
      echo "*** Thread detail"
      grep -A 20 "nid=${FIRST_TID_HEX}" ${OUTFILE_TDUMP}
    } >>"${OUTFILE_REPORT}"
  else
    echo "No thread with CPU>5% in sample --> no report"
  fi
  
  if ((--COUNTER>0))
  then
    echo "Waiting 5s"
    sleep 5
  fi
done