splice hacks

cron splice

# cron changes

# Puppet Name: tiles_monitor_splice_tiles_index_crawl
SPLICE_SETTINGS=splice_config.SpliceConfig
* * * * * /usr/local/bin/splice_tiles_index_crawl.sh 2>&1 | logger -t splice_tiles_index_crawl

# due a bug in splice_tiles_index_crawl.sh failing sometimes, (greenlets, ew)
* * * * * /usr/local/bin/splice_tiles_watchdog.sh 180 | logger -t splice_tiles_index_crawl

splice_tiles_index_crawl.sh

#!/bin/bash

LOCK_FILE=/root/splice_tile_lock.tmp

if [ -e $LOCK_FILE ]; then
    echo "LOCK FILE exists, ending early"
    exit 2
fi

echo $(date +%s) > $LOCK_FILE

RESULTS="$(/opt/splice/bin/python /opt/splice/bin/tile_index_crawl.py -v)"
SUCCESS="$(echo "$RESULTS" | grep -c '^SUCCESS')"
ERROR="$(echo "$RESULTS" | grep -c '^ERROR')"

RUNNING=$(expr $(date +%s) - $(cat $LOCK_FILE))
echo "successes: $SUCCESS, errors: $ERROR, took: $RUNNING seconds"

python -c "from datadog import statsd; statsd.gauge('splice.tiles_index_crawl.success', $SUCCESS)"
code="$?"
python -c "from datadog import statsd; statsd.gauge('splice.tiles_index_crawl.error', $ERROR)"

if [ -e $LOCK_FILE ]; then
    rm $LOCK_FILE
fi

exit $(($code|$?))

splice_tiles_watchdog.sh

#!/bin/sh

LOCK_FILE=/root/splice_tile_lock.tmp
MAX_RUN_TIME=$1

if [ ! -e $LOCK_FILE ]; then
    exit 0
fi

# figure out how long its been running for
RUNNING=$(expr $(date +%s) - $(cat $LOCK_FILE))

if [ $RUNNING -gt $MAX_RUN_TIME ]; then
    INDEX_PID=$(ps ax | grep tile_index_crawl.py | grep -v grep  | awk '{print $1}')
    echo "(WATCHDOG) PID $INDEX_PID, running for $RUNNING seconds. Killing it."
    kill $INDEX_PID
    if [ -e $LOCK_FILE ]; then
        rm $LOCK_FILE
    fi
fi