Extremly helpfull scripts for debugging c++ application on linux.

parse_crashdump.sh

#!/bin/bash

# This script parses crashdump and print lines from source files

LOGFILE=$1

NUM_SRC_CONTEXT_LINES=3

old_IFS=$IFS  # save the field separator           
IFS=$'\n'     # new field separator, the end of line           

for bt in `cat $LOGFILE |c++filt| sed -e '1,/Backtrace:/ d; /Memory map:/,$ d'`; do
   IFS=$old_IFS     # restore default field separator 
   EXEC=`echo $bt | cut -d'(' -f1`  
   ADDR=`echo $bt | cut -d'[' -f2 | cut -d']' -f1`
   A2L=`addr2line -a $ADDR -e $EXEC -pfC`
   #echo "A2L:        $A2L"

   FUNCTION=`echo $A2L | sed 's/\<at\>.*//' | cut -d' ' -f2-99`
   FILE_AND_LINE=`echo $A2L | sed 's/.* at //'`
   if [[ "$FILE_AND_LINE" =~ "??"  ]]; then
      echo "BACKTRACE:  $EXEC $ADDR"
      continue
   fi 
   #echo "BACKTRACE:  $EXEC $ADDR"
   printf '\n'
   echo "FILE:       $FILE_AND_LINE"
   echo "FUNCTION:   $FUNCTION"

   # print offending source code
   SRCFILE=`echo $FILE_AND_LINE | cut -d':' -f1`
   LINENUM=`echo $FILE_AND_LINE | cut -d':' -f2`
   if ([ -f $SRCFILE ]); then
      cat -n $SRCFILE | grep -C $NUM_SRC_CONTEXT_LINES "^ *$LINENUM\>" | sed "s/ $LINENUM/*$LINENUM/"
   else
      echo "File not found: $SRCFILE"
   fi
   IFS=$'\n'     # new field separator, the end of line           
   printf '\n'
done

IFS=$old_IFS     # restore default field separator 

run.sh

#!/bin/sh

# This script preload libSegFault and start application. 
# It create crashdump on SIGABRT or SIGSEGV signals

set -u

# NOTE: project has to be compiled with -g and linked with -rdynamic

PROGNAME=$1

# Take the first library
LIBPATH=`ldconfig -p | grep libSegFault.so | sed -e '2,$d;s/.*=> //'`

env SEGFAULT_USE_ALTSTACK='True' SEGFAULT_SIGNALS="abrt segv" SEGFAULT_OUTPUT_NAME="/tmp/$PROGNAME.crash" LD_PRELOAD="$LIBPATH/libSegFault.so" $@

send_crashdump.sh

#!/bin/bash

# This script monitor specified directory. When file with crashdump appears 
# script package crashdump and binaries listed in it, create new ticket 
# in Redmine and attach package to it

set -e 
set -u

BIN_PREFIX="" # add / at the end

# Redmine access token
RED_KEY="613116ecbb51f17a1e397967dcee8ff9cdcfcf9e"

# Redmine url
RED_URL="http://redmine.local"

# Redmine project id (not the project unique id)
RED_PRJ=56

# Directory where reports will be copied
CEMENTRY_DIR="/tmp/cementry"

# Binaries black list
BLACK_BINS="libc.so libQt libpthread"

DIR_TO_WATCH=$1
TMP_FILE=`mktemp`
TMP_ARCHIVE=`mktemp -d`

DEBUG=true

decho() {
    if [[ $DEBUG == "true" ]]; then
        echo $@
    fi
}

copyBins() {
    local LOGFILE="$1"
    local DEST="$2"
    local old_IFS=$IFS  # save the field separator           
    IFS=$'\n'     # new field separator, the end of line           

    for bt in `cat $LOGFILE |c++filt| sed -e '1,/Backtrace:/ d; /Memory map:/,$ d'`; do
        local EXEC=`echo $bt | cut -d'(' -f1`  
        local Skip="false"
        IFS="$old_IFS"
        for i in $BLACK_BINS; do
            if [[  "${BIN_PREFIX}$EXEC" =~ "$i" ]]; then
                Skip="true"
                break
            fi
        done
        IFS=$'\n'     # new field separator, the end of line           

        if [[ "$Skip" == "false" ]]; then
            decho "cp ${BIN_PREFIX}$EXEC  $DEST/bin"
            cp "${BIN_PREFIX}$EXEC"  "$DEST/bin"
        fi
        
    done

    IFS=$old_IFS     # restore default field separator 
}


reportRedmine() {
    local Grave="$1"
    local Filename=`basename $Grave`

    local Token=`curl -s --data-binary "@$Grave" -H "Content-Type: application/octet-stream" -X POST \
        -H "X-Redmine-API-Key: $RED_KEY" "$RED_URL"/uploads.xml  | grep token | sed 's|.*<token>||;s|</token>.*||' `
    local Issue=`mktemp`
    decho "Token for upload is $Token"
cat > $Issue << EOF 
    <?xml version="1.0" encoding="ISO-8859-1" ?>
    <issue>
      <subject>New gravestone</subject>
      <project_id>$RED_PRJ</project_id>
      <uploads type="array">
        <upload>
          <token>$Token</token>
          <filename>$Filename</filename>
          <content_type>application/x-gzip</content_type>
        </upload>
      </uploads>
    </issue>
EOF

    local Ans=`curl -s -v -H "Content-Type: application/xml" -X POST --data "@$Issue"  -H "X-Redmine-API-Key: $RED_KEY" "$RED_URL"/issues.xml`
    decho $Ans
}


mkdir -p "$CEMENTRY_DIR" || true

# while inotifywait -q -r -e modify -e create  --format '%w%f' \
while inotifywait -q -r -e modify  --format '%w%f' \
 "$DIR_TO_WATCH" > "$TMP_FILE"
do
    sleep 0.5 # waiting for full crash dump

    File=`cat $TMP_FILE`
    DATE=`date +'%y.%m.%d_%H.%M.%S'`

    GRAVE_BASENAME="grave_$DATE"
    GRAVE_PATH="$TMP_ARCHIVE/$GRAVE_BASENAME"
    GRAVESTONE="$CEMENTRY_DIR/$GRAVE_BASENAME.tbz"

    mkdir -p $GRAVE_PATH/bin

    decho "================================================================================"
    cp "$File" "$GRAVE_PATH/crashdump"
    copyBins "$File" "$GRAVE_PATH"

    (
        cd "$TMP_ARCHIVE"
        tar -cjf "$GRAVESTONE" "./$GRAVE_BASENAME" 
    )

    reportRedmine "$GRAVESTONE"

    rm -rf $GRAVE_PATH
done