a tool to detect updated wsp files, useful for finding statsd daemons still writing metrics to graphite

find-updated-wsps.sh

#!/bin/bash

# find-updated-wsp.sh: a tool to detect updated wsp files, useful for finding statsd daemons still writing metrics to graphite;
# especially useful when you tell statsd to write to new graphite servers, and you want to make sure nothing is writing data to the old graphite machine.
# why this way:
# * you can't just check for network traffic, statsd will stay connected even when it has no data to send
# * plain mtime of wsp files is not sufficient: statsd will keep writing the last known value to gauges, and 0's for inactive counters, this data should be ignored.
# * both statsd and graphite write stats of their own, updates of these files can be ignored.

# this script has two modes of operation, based on $1:
# $1 <filename>: just process this filename, and give some extra output.
# $2 anything else: find all files that have received any relevant updates

# from when on do you want to find updated data (i.e. date after supposedly switching graphite machines)
newer_date="2012-10-04 00:00:00 EDT"
# whisper storage dir
whisper_dir=/graphitemart/storage/whisper


# don't edit from this point. unless you want to
get_nonzero_entries () {
	local f=$1
	/usr/local/bin/python2.6 /usr/local/bin/whisper-fetch.py --from=$newer_date_ts $f | cut -f2 | grep -v '^0.000000$' | grep -v ^None$
}
has_relevant_updates () {
	local f=$1
	lines=
	egrep -q "^$whisper_dir/(stats/statsd|carbon)" <<< "$f" && return 1 # ignore statsd/carbon's own updates
        lines=$(get_nonzero_entries $f | wc -l)
	[ $lines -eq 0 ] && return 1 # statsd has been writing nothing but 0.0000000 values to it, this seems safe to ignore (probably an inactive counter)
	[[ $f =~ ^$whisper_dir/stats/gauge ]] && [ $(get_nonzero_entries $f | uniq | wc -l) -lt 2 ] && return 1 # a gauge with only 1 unique file -> stale number that statsd keeps writing, ignore
	return 0
}

newer_date_ts=$(date -d "$newer_date" +%s)
tmpfile=$(mktemp /tmp/tmp.find-updated-wsp.XXXXX)
touch -m  -d "$newer_date" $tmpfile

if [ -n "$1" -a -f "$1" ]; then
	f=$1
	has_relevant_updates $f && echo "$f $lines" && get_nonzero_entries $f | uniq
else
	while read f; do
		has_relevant_updates $f && echo "$f $lines"
	done < <(find $whisper_dir -type f -name '*.wsp' -newer $tmpfile)
fi