Start, stop, and back up Open Satellite Project goesdump and xritdemod

osp-handler.sh

#!/bin/bash
###AUTHOR###
#Harry Dove-Robinson 2018-02-08
#[email protected]
#https://gist.github.com/hdoverobinson
#https://github.com/hdoverobinson

###NOTES###
#Used by wx-star.com for 24/7 operation of Open Satellite Project software for HRIT/EMWIN direct readout
#Manages OSP processes inside GNU Screen sessions, performs daily backups and rotation of OSP output data
#Packages installed: cpulimit, diffutils, pigz, screen
#It is recommended to set the system timezone to UTC

###USAGE###
#When this script is run at least once per 24 hours:
#-OSP processes will be restarted
#-Up to 48 hours of OSP output data will be stored ephemerally in $OUTPUT_DIR
#-Exactly 24 hours of OSP output data created between midnight today and midnight yesterday will be backed up as tar.gz in $ARCHIVE_DIR/$YESTERDAY_YEAR
#Accepts arguments "stop", "rotate", and "start". When no arguments are provided the script will run all three functions in that order. Rotation will be performed in background to minimize downtime and after
#a waiting period so OSP can finish writing its output from the previous day
#Accepts options "hard" and "verbose" to send SIGKILL to processes instead of SIGINT, or to send all command output to stdout via "set -x", respectively
#Examples:
#"./osp-handler" to stop all OSP processes, attempt to backup and rotate OSP output data, and start OSP processes again
#"./osp-handler stop hard verbose" to only send SIGKILL to all OSP processes and show output verbosely
#"./osp-handler rotate" to only attempt to backup and rotate OSP output data with minimal output to stdout
#"0 */4 * * * root /bin/bash -c '/root/osp-handler.sh > /var/log/osp-handler.log 2>&1'" in a cron.d file to run all functions every 4 hours and store output to logfile

#where daily tar.gz archives will be stored
ARCHIVE_DIR="/media/data/archive"
#where xritdemod binaries are located
XRITDEMOD_DIR="/root/osp-build/bin/xritdemod"
#where goesdump binaries are located
GOESDUMP_DIR="/root/osp-build/bin/goesdump"
#goesdump output directory
OUTPUT_DIR="$GOESDUMP_DIR/output"

#number of days to retain tar.gz archives
ARCHIVE_HOLD="365"
#total CPU percentage to limit backup processes
CPU_LIMIT="100"
#time to wait for OSP to finish writing yesterday's files before attempting a backup
WAIT_TIME="30m"
#time between process stop and start attempts
SLEEP_TIME="1s"

cpulimit_wait ()
{
#force a wait for cpulimit processes to close before proceeding
PID=$(ps -eo pid,comm,args | grep -w "cpulimit" | grep -v "grep" | awk '$2 == "cpulimit"' | grep "$1" | awk '{print $1}') &&
while kill -0 $PID > /dev/null 2>&1
do
sleep "$SLEEP_TIME"
done &&
wait
}

stop ()
{
echo "$(tput setaf 2 2> /dev/null)Stopping OSP...$(tput sgr0 2> /dev/null)" &&
GOESDUMP=$(ps -eo pid,comm,args | grep -w "./goesdump" | grep -v "grep" | awk '$2 == "goesdump"' | awk '{print $1}') &&
XRITDECODER=$(ps -eo pid,comm,args | grep -w "./xritDecoder" | grep -v "grep" | awk '$2 == "xritDecoder"' | awk '{print $1}') &&
XRITDEMODULATOR=$(ps -eo pid,comm,args | grep -w "./xritDemodulator" | grep -v "grep" | awk '$2 == "xritDemodulator"' | awk '{print $1}') &&
while kill -0 $GOESDUMP > /dev/null 2>&1
do
kill -$SIG $GOESDUMP > /dev/null 2>&1 && sleep "$SLEEP_TIME"
done &&
while kill -0 $XRITDECODER > /dev/null 2>&1
do
kill -$SIG $XRITDECODER > /dev/null 2>&1 && sleep "$SLEEP_TIME"
done &&
while kill -0 $XRITDEMODULATOR > /dev/null 2>&1
do
kill -$SIG $XRITDEMODULATOR > /dev/null 2>&1 && sleep "$SLEEP_TIME"
done &&
sync &&
echo 3 > /proc/sys/vm/drop_caches &&
if [[ $(screen -ls | wc -l) -gt 2 ]]
then
screen -wipe > /dev/null 2>&1
fi &&
wait
}

rotate ()
{
TODAY="$(date -u '+%F' --date="today")" &&
YESTERDAY="$(date -u '+%F' --date="yesterday")" &&
YESTERDAY_YEAR="$(date -u '+%Y' --date="yesterday")" &&
YESTERDAY_MONTH="${YESTERDAY_YEAR}-$(date -u '+%m' --date="yesterday")" &&
MAX_ARCHIVE_DAY="$(date -u '+%F' --date="today -$ARCHIVE_HOLD days")" &&

if [[ -d "$OUTPUT_DIR" ]] && ! [[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" ]]

then
echo "$(tput setaf 2 2> /dev/null)Waiting for $WAIT_TIME before starting backup...$(tput sgr0 2> /dev/null)" &&
sleep "$WAIT_TIME" &&
echo "$(tput setaf 2 2> /dev/null)Archiving files in $OUTPUT_DIR from $YESTERDAY to $ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz...$(tput sgr0 2> /dev/null)" &&
mkdir -p "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY" &&
FIND_CMD="find "$OUTPUT_DIR" \( -newermt "$YESTERDAY" ! -newermt "$TODAY" \) -type f ! -name '*dup*' ! -name '*.zip' ! \( -path '*/EMWIN/*' -name '*.lrit' \)" &&

#multithreaded tar.gz backup of files from $FIND_CMD, print $FIND_CMD output to list
tar -cf - -C "$(realpath "$OUTPUT_DIR/../")" --files-from=<(eval "$FIND_CMD 2> /dev/null" | sed "s@$(realpath "$OUTPUT_DIR/../")/@@g" | tee "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list") | cpulimit -q -l "$CPU_LIMIT" -- pigz > "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" &&
cpulimit_wait pigz &&

#compare tar.gz contents to files in $OUTPUT_DIR, print matches to list
tar -dzvf "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" -C "$(realpath "$OUTPUT_DIR/../")" > "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.list" &&

#make sha256 checksum of tar.gz backup
cpulimit -q -l "$CPU_LIMIT" -- sha256sum "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" > "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.sha256" &&
cpulimit_wait sha256sum &&

#check if all backup files are present, compare list of source files to list of files inside the archive
if [[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" ]] && [[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list" ]] && \
[[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.list" ]] && [[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.sha256" ]] && \
SRC_FILES="$(cat "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list" | wc -l)" && \
BACK_FILES="$(cat "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.list" | wc -l)" && \
#filename comparison excludes EMWIN because of possible bug in tar where underscores in long EMWIN filenames were sometimes printed as \022 but were printed normally in source
cmp -s <(cat "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list" | grep -v '/EMWIN/' | sort -n) <(cat "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.list" | grep -v '/EMWIN/' | sort -n) && \
[[ "$SRC_FILES" -gt 0 ]] && [[ "$BACK_FILES" -gt 0 ]] && [[ "$SRC_FILES" -eq "$BACK_FILES" ]]

then
#delete source file list and old data if archive valid
rm "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list" &&
echo "$(tput setaf 2 2> /dev/null)Archive $ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz complete!$(tput sgr0 2> /dev/null)" &&
echo "$(tput setaf 2 2> /dev/null)Deleting files from $OUTPUT_DIR older than $YESTERDAY...$(tput sgr0 2> /dev/null)" &&
find "$OUTPUT_DIR" ! -newermt "$YESTERDAY" -type f -delete &&
echo "$(tput setaf 2 2> /dev/null)Deleting archives from $ARCHIVE_DIR/$YESTERDAY_YEAR older than $MAX_ARCHIVE_DAY...$(tput sgr0 2> /dev/null)" &&
find "$ARCHIVE_DIR" ! -newermt "$MAX_ARCHIVE_DAY" -type f -delete &&
echo "$(tput setaf 2 2> /dev/null)Archive and data rotation complete!$(tput sgr0 2> /dev/null)"

else
#delete attempted archive files if failed
echo "$(tput setaf 1 2> /dev/null)Archive failed: $ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.$(tput sgr0 2> /dev/null)" &&
if [[ -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" ]]
then
echo "$(tput setaf 1 2> /dev/null)Deleting failed archive: $ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.$(tput sgr0 2> /dev/null)" &&
rm -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz" &&
rm -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.filesource.list" &&
rm -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.list" &&
rm -f "$ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.sha256"
fi &&
wait &&
exit 1
fi

else
echo "$(tput setaf 3 2> /dev/null)Archive for $YESTERDAY already exists at $ARCHIVE_DIR/$YESTERDAY_YEAR/$YESTERDAY_MONTH/$YESTERDAY/$YESTERDAY.tar.gz.$(tput sgr0 2> /dev/null)"
fi
}

start ()
{
echo "$(tput setaf 2 2> /dev/null)Starting OSP...$(tput sgr0 2> /dev/null)" &&
screen -dmS xritDemodulator bash -c "cd $XRITDEMOD_DIR/ && ./xritDemodulator" &&
sleep "$SLEEP_TIME" &&
screen -dmS xritDecoder bash -c "cd $XRITDEMOD_DIR/ && ./xritDecoder" &&
sleep "$SLEEP_TIME" &&
screen -dmS goesdump bash -c "cd $GOESDUMP_DIR/ && ./goesdump"
}

echo "$(tput setaf 2 2> /dev/null)$(date -u '+%F %R:%S UTC') Starting $(basename "$0")$(tput sgr0 2> /dev/null)" &&

if ! command -v cmp > /dev/null 2>&1 || ! command -v cpulimit > /dev/null 2>&1 || ! command -v pigz > /dev/null 2>&1 || ! command -v screen > /dev/null 2>&1
then
echo "$(tput setaf 2 2> /dev/null)Installing requisite packages...$(tput sgr0 2> /dev/null)" &&
apt-get -qq update &&
apt-get -qq install cpulimit diffutils pigz screen
fi &&

if [[ "$@" =~ .*verbose.* ]]
then
set -x
else
set +x
fi &&

if [[ "$@" =~ .*hard.* ]]
then
SIG=9
else
SIG=2
fi &&

if [[ "$@" =~ .*stop.* ]]
then
stop
elif [[ "$@" =~ .*rotate.* ]]
then
rotate
elif [[ "$@" =~ .*start.* ]]
then
start
else
stop &&
(rotate &) &&
start
fi &&

wait &&
exit