Backups all distinguished Poirot entries in a monthly index and deletes the individual per-day indices

backup-poirot.sh

#! /bin/bash
# Invoke using YYYY.MM of indices to backup

# exit on uncaught error
set -e

# non zero exit status on a pipeline causes the whole pipeline to fail
set -o pipefail

function backup_index {
  INDEX_TO_BACKUP=$1
  ARCHIVE_INDEX=$2
  FILE_NAME=$3
  TEMP_FILE="$OUTPUTDIR/$FILE_NAME.json"
  PARSED_FILE="$OUTPUTDIR/$FILE_NAME.json.parsed"
  COUNT=$(curl -f -s -XGET "$ES_URL/$INDEX_TO_BACKUP/_search?search_type=count" -d "$QUERY" | sed -E 's/^.*"total":([0-9]+).*/\1/g')

  if [ $? -ne 0 ]; then
    echo "Could not retrieve document count for index. This could mean the index does not exist."
    return 1
  fi

  if [ $COUNT -le 0 ]; then
    echo "No archive entries to archive."
    return 0
  fi

  echo "Retrieving $COUNT entries to archive..."

  curl -f -s -XGET "$ES_URL/$INDEX_TO_BACKUP/activity/_search?size=$COUNT" -d "$QUERY" > "$TEMP_FILE"

  cp "$TEMP_FILE" "$TEMP_FILE.copy" && gzip $TEMP_FILE && mv "$TEMP_FILE.copy" "$TEMP_FILE"

  if [ $? -ne 0 ]; then
    echo "Failure performing backup"
    return 1
  fi

  echo "Index archived in file $TEMP_FILE.gz"

  echo "Parsing retrieved JSON..."
  cat $TEMP_FILE | /usr/local/bin/jq --compact-output ".hits.hits[] | {create :{_index: \"$ARCHIVE_INDEX\", _type: ._type}}, ._source" > "$PARSED_FILE"

  if [ $? -ne 0 ]; then
    echo "Failure parsing JSON."
    return 1
  fi
  echo "JSON parsed."

  echo "Creating the new archive index..."
  curl -f -s -XPOST "$ES_URL/_bulk" --data-binary @$PARSED_FILE > /dev/null

  CURL_RETURN_STATUS=$?

  rm "$TEMP_FILE" "$PARSED_FILE"

  if [ $CURL_RETURN_STATUS -ne 0 ]; then
    echo "Failure saving archive index."
    return 1
  fi
}

function delete_index {
  INDEX=$1

  echo "Deleting index $INDEX..."
  curl -f -s -XDELETE "$ES_URL/$INDEX/" > /dev/null

  if [ $? -ne 0 ]; then
    echo "Failure deleting index"
    return 1
  fi

  echo "Index successfully deleted"
}

############################################################################

ES_URL='localhost:9200'
INDEX="poirot-$1*"
ARCHIVE_INDEX="poirot-archive-$1"
OUTPUTDIR='/home/ubuntu/elasticsearch-archives'

# Backup Verboice call log entries
QUERY='
{
  "query": {
    "constant_score": {
      "filter": {
        "or": [
          { "exists": { "field": "call_log_id" } },
          { "exists": { "field": "step_type"   } }
        ]
      }
    }
  }
}'

echo "Will try to archive index $INDEX"

mkdir -p $OUTPUTDIR

backup_index $INDEX $ARCHIVE_INDEX $ARCHIVE_INDEX && delete_index $INDEX

if [ $? -ne 0 ]; then
  echo "Something went wrong..."
  exit 1
fi

echo "Done!"