Script to clean up incomplete Elasticsearch snapshots (assuming fs)

es-snapshots-cleanup.sh

#!/bin/bash

# repository for snapshots
snapshots_repo="es-snapshots"

# filesystem for snapshots
snapshots_fs="/es-snapshots"

# elasticsearch url
elasticsearch_url="http://esdatafb101.athenahealth.com:9210"

function usage()
{
    echo "Script to clean up incomplete Elasticsearch snapshots (assuming fs)"
    echo ""
    echo "$0"
    echo -e "\t-h --help"
    echo -e "\t--elasticsearch-url=$elasticsearch_url"
    echo "\t--snapshots-fs=$snapshots_fs"
    echo "\t--snapshots-repo=$snapshots_fs"
    echo ""
}

while [ "$1" != "" ]; do
    PARAM=`echo $1 | awk -F= '{print $1}'`
    VALUE=`echo $1 | awk -F= '{print $2}'`
    case $PARAM in
        -h | --help)
            usage
            exit
            ;;
        --elasticsearch-url)
            elasticsearch_url=$VALUE
            ;;
        --snapshots-fs)
            snapshots_fs=$VALUE
            ;;
        --snapshots-repo)
            snapshots_repo=$VALUE
            ;;
        *)
            echo "ERROR: unknown parameter \"$PARAM\""
            usage
            exit 1
            ;;
    esac
    shift
done

# ensure tools are availabel
command -v jq >/dev/null 2>&1 || \
  { 
    echo "I require jq but it's not installed. Aborting." 1>&2
    exit 1
  }
command -v curl >/dev/null 2>&1 || \
  { 
    echo "I require curl but it's not installed. Aborting." 1>&2
    exit 1
  }

# determine disk space usage of filesystem for snapshots
snapshots_fs_usage=$( df -P ${snapshots_fs} | awk '/[0-9]%/{print $(NF-1)}' | cut -d% -f1 )

if [ $snapshots_fs_usage -le 98 ]; then
  echo "Not running out of space \"${snapshots_fs} (${snapshots_fs_usage}%)\" on $(hostname) as on $(date)" 1>&2
  exit 1
fi

# find all unsuccessful snapshots
unsuccessful_snapshots=$(
  curl -s ${elasticsearch_url}/_snapshot/${snapshots_repo}/_all | \
  jq -r '.snapshots[] | select(.state!="SUCCESS") | .snapshot' \
)

if [ -z "${unsuccessful_snapshots}" ]; then
  "No incomplete snapshots found. Please try other cleanup methods." 1>&2
  exit 1
fi

echo "Found incomplete snapshots for ES at ${elasticsearch_url}."
echo "Please hang on, this can take a while as our hamsters are pretty lazy."
echo ""

for snapshot in $unsuccessful_snapshots
do
  echo -n "Deleting incomplete snapshot ${snapshot} using ES API ... "
  curl -s -XDELETE ${elasticsearch_url}/_snapshot/${snapshots_repo}/${snapshot}
  echo ""
done

# determine disk space usage of filesystem for snapshots
snapshots_fs_usage=$( df -P ${snapshots_fs} | awk '/[0-9]%/{print $(NF-1)}' | cut -d% -f1 )

echo ""
echo "Not running out of space any more hopefully at \"${snapshots_fs} (${snapshots_fs_usage}%)\" on $(hostname) as on $(date)" 1>&2

Expected output:

Found incomplete snapshots for ES at http://esdatafb101.athenahealth.com:9210.
Please hang on, this can take a while as our hamsters are pretty lazy.

Deleting snapshot curator-tag-20161206021336 using ES API... {"acknowledged":true}

Not running out of space any more hopefully at "/es-snapshots (84%)" on esdatafb100.athenahealth.com as on Wed Dec  7 15:03:48 EST 2016