Script to remove documents from elasticsearch by query or whole index

purge_elasticsearch.sh

#!/bin/bash
# elasticsearch cleanup  script

ES_BASEDIR='/var/lib/elasticsearch/'
ES_HTTP='http://localhost:9200'
RETENTION_PERIOD='365'
QUERY=''
REMOVE=false
TEMP_FILE=/tmp/_usr_local_sbin_purge_elasticsearch.sh_$$

usage(){
cat <<EOF
usage: $0 <options>

  -q query to clean
  -d number of days to keep content
  -h show this help message
  -r remove whole index
  -p set path to elasticsearch index dir
  -u set url to elasticsearch http interface

  Examples:
    ./purge_elasticsearch.sh -u 'http://localhost:9200' -d 10 -p '/var/lib/es' -q '"match": { "syslog_severity": { "query": "debug", "type": "phrase" } }'
    ./purge_elasticsearch.sh -d 60 -q '"match": { "_id": { "query": "AVObSq6sPYpCdtDyj_yE", "type": "phrase" } }'
    ./purge_elasticsearch.sh -d 90 -r

EOF
}


while getopts 'u:d:p:q:r:h' OPTION
do
  case $OPTION in

    u) ES_HTTP=$OPTARG;;
    d) RETENTION_PERIOD=$OPTARG;;
    p) ES_BASEDIR=$OPTARG;;
    q) QUERY=$OPTARG;;
    r) REMOVE=true;;
    h) usage && exit 0;;
  esac
done

INDICES=`find "$ES_BASEDIR" -type d -name 'logstash-*' -ctime +$RETENTION_PERIOD | sed 's#.*/##'`
[ -z "$INDICES" ] && echo 'No index found!'

for INDEX in $INDICES; do
  echo -en "\nindex: $INDEX"

  if "$REMOVE"; then

    # remove whole index
    curl -s -XDELETE "$ES_HTTP/$INDEX/" &>/dev/null

    echo -n ' removed!'

  elif [ -n "$QUERY" ]; then
    which jq &>/dev/null || { echo 'jq is not installed'; exit 2; }

    # It queries for the data and then delete them by _bulk method.
    # 
    # I use this complicated way because delete-by-query is deprecated in ES 2.0+
    #   https://www.elastic.co/guide/en/elasticsearch/reference/1.6/docs-delete-by-query.html

    echo 'This file cannot be empty to run the while loop at least once' > $TEMP_FILE
    while [ -s $TEMP_FILE ]; do

      # size: 10000 is maximum of '_bulk' method atm
      # this curl will look for proper documents which are evetually 
      # transformed to proper format for _buld by 'jq' program. The example
      # of one line:
      #
      #   {"delete":{"_index":"logstash-2016.05.03","_type":"syslog","_id":"AVR0IQUiPYpCdtDyUCMj"}}
      #
      curl -s "$ES_HTTP/$INDEX/_search?pretty" -d "{ \"size\": 10000, \"fields\": [], \"query\": { $QUERY } }" 2>/dev/null |
        jq -c '.hits.hits | .[] | { delete: { _index: ._index, _type: ._type, _id: ._id  } }' 2>/dev/null > $TEMP_FILE

      # check if at least one document was found
      [ -s $TEMP_FILE ] || { echo -n ' done!'; echo 'To run loop at least once' > $TEMP_FILE; break ; }
      echo >> $TEMP_FILE      # _bulk expect newline at the end

      # delete all found data
      curl -XPOST $ES_HTTP/$INDEX/_bulk?pretty --data-binary @$TEMP_FILE &>/dev/null

      # you have to refresh otherwise the search method can find deleted documents again
      curl -XPOST "$ES_HTTP/$INDEX/_refresh" &>/dev/null

      echo -n '.'

    done

  else

    echo -e "\n\nYou have to specify the query by -q '<query>' or -r to remove whole index.\n\nNothing done!"
    exit 1

  fi
done

rm -f $TEMP_FILE
echo -e "\nDONE"