script to delete old indexes in Elasticsearch

elk_remove-old-indexes.sh

#! /bin/bash
# Francois Bayart - 04 Feb 2015
# script to delete old indexes in Elasticsearch
#
# To keep some specific date, it's possible to add it in DATE_TO_KEEP variable (one date per line, format YYYY.MM.DD)
# Take care about to always use 2 digits for days and months
#

HOW_MANY_DAYS_TO_KEEP=10

ELASTICSEARCH_URL="http://localhost:9200"

DATE_TO_KEEP="
2015.01.24
2015.01.25
"

DATE_TO_REMOVE_UNIXTIMESTMAP=$(date -d  "${HOW_MANY_DAYS_TO_KEEP} days ago" '+%s')
ALL_ES_INDEXES=`curl -s ${ELASTICSEARCH_URL}/_cat/indices?v | awk -F" " '{if ($3 ~ /^log|^\.marvel/) print $3}' | sort -u -t"-" -k1`

for f in $ALL_ES_INDEXES
do
  IS_IT_OK=true
  ES_INDEX_DATE=$(echo "$f" | grep -Po ".*-\K([0-9]{4}.[0-9]{2}.[0-9]{2})\.*")
  ES_INDEX_DATE_UNIXTIMESTAMP=$(date -d "${ES_INDEX_DATE//./} 01" '+%s')
  if [ $ES_INDEX_DATE_UNIXTIMESTAMP -lt $DATE_TO_REMOVE_UNIXTIMESTMAP ]
  then
    for tokeep in $DATE_TO_KEEP
    do
      if [ $ES_INDEX_DATE_UNIXTIMESTAMP -eq $(date -d "${tokeep//./} 01" '+%s') ]
      then
        IS_IT_OK=false
      fi
    done

    if [ $IS_IT_OK = true ]
    then
      echo "curl -XDELETE ${ELASTICSEARCH_URL}/$f"
      if [[ $1 == "-delete" ]]
      then
        curl -s -XDELETE ${ELASTICSEARCH_URL}/$f
      fi
    else
      echo "please keep $f"
    fi
  fi
done