Example coredump uploader

coredump_uploader.sh

#!/bin/bash
# Depends on zstd (>1.3.6) and aws-cli
set -o errexit
set -o nounset
set -o pipefail

# Set kernel.core_pattern = | coredump_uploader.sh %P %s %E
PID=$1
shift
SIGNAL=$1
shift
EXECUTABLE=$1
shift

# Set these to appropriate values for your environment
readonly BUCKET=
readonly APP_NAME=
readonly INSTANCE_ID=
readonly COREDUMP_BACKOFF_MINS=60

TARGET_STDERR="/proc/${PID}/fd/2"
if ! exec 2>>"${TARGET_STDERR}" >&2; then
  # Redirect output into the target's stderr, if possible
  exec 2> >(exec 2>&-; logger -t "coredump-uploader-${PID}")
  exec >&2
fi

echo "${0##*/} initiated for PID ${PID}"
mkdir -p /run/coredumps

find /run/coredumps -mmin "+${COREDUMP_BACKOFF_MINS}" -type f -delete
LAST=$(find /run/coredumps -type f -printf "%t\n" | head -n1)
if [ -n "${LAST}" ]; then
   # Back off without creating a file
   echo "Last core dump was at ${LAST}; too soon to upload another."
   exit
fi
mktemp -p /run/coredumps dump.XXXXXXXXXX

MACHINE_DATE=$(date "+%s.%N")
HUMAN_DATE=$(date -d "@${MACHINE_DATE}" '+%Y-%m-%dT%H:%M:%S.%N')
TWO_63=9223372036854775808
REVERSE_DATE=$(($TWO_63 - ${MACHINE_DATE/.})) # overflows in 2230

S3_KEY="s3://${BUCKET}/apps/${APP_NAME}/coredumps/${INSTANCE_ID}/${REVERSE_DATE}_${HUMAN_DATE}"
echo "Uploading to: ${S3_KEY}"

echo "Uploading metadata"
EXE_HASH=$(sha256sum /proc/$PID/exe | cut -d' ' -f1)
(
    cat <<EOF
Date: $HUMAN_DATE
PID: $PID
Signal: $SIGNAL
Command: ${EXECUTABLE//!//}
SHA256(Executable): $EXE_HASH
EOF

  lsof -p $PID -Pn
) | aws s3 cp - "${S3_KEY}/meta"

echo "Uploading tarball"
#XXX: which of these files duplicate data already present in the core dump?
#XXX: also, if any of these files are useful, should we get the same from each
# running thread
tar -C /proc/$PID -c \
  cgroup \
  cmdline \
  cpuset \
  comm \
  environ \
  gid_map \
  io \
  maps \
  mountinfo \
  sched \
  schedstat \
  smaps \
  stack \
  stat \
  status \
  uid_map \
  wchan \
| aws s3 cp - "${S3_KEY}/proc.tar"

echo "Uploading core file"
echo "You may see 'Aborted (Core Dumped)' printed to the terminal before completion"
# If you can't use zstd, consider lz4 instead: it has compression throughput
# suitable for streaming uploads.
zstd --adapt | aws s3 cp - "${S3_KEY}/core.zst"