ElasticSearch log index backup & restore scripts

backup.sh

#!/bin/bash
# herein we backup our indexes! this script should run at like 3 AM every first day of the month, after logstash
# rotates to a new ES index and theres no new data coming in to the old one. we grab metadatas,
# compress the data files and backs up to whatever long term storage.

set -x

. ./config.sh

echo "Checking that index exist in ES"
if [ `curl -sI $ESURL/$INDEXNAME | grep OK | wc -l` -eq 0 ]
then
  echo "Index $INDEXNAME doesn't exist, nothing to backup"
  exit 0
fi

# create mapping file with index settings. this metadata is required by ES to use index file data
echo -n "Backing up metadata of index $INDEXNAME ... "
curl -XGET -o $TMPDIR/mapping $ESURL"/$INDEXNAME/_mapping?pretty=true" > /dev/null 2>&1
sed -i '1,2d' $TMPDIR/mapping #strip the first two lines of the metadata

# setting도 백업 필요
curl -XGET -o $TMPDIR/settings $ESURL"/$INDEXNAME/_settings?pretty=true" > /dev/null 2>&1
head -n -2 $TMPDIR/settings | sed '1,2d' > $TMPDIR/settings_cut

echo '{' > $MAPPING
cat $TMPDIR/settings_cut >> $MAPPING
echo ', "mappings": {' >> $MAPPING

# echo '{"settings":{"number_of_shards":5,"number_of_replicas":1},"mappings":{' > $MAPPING
# prepend hardcoded settings metadata to index-specific metadata
cat $TMPDIR/mapping >> $MAPPING
echo "DONE!"

# now lets tar up our data files. these are huge, so lets be nice
echo -n "Backing up data files of index $INDEXNAME (this may take some time) ... "
mkdir -p $BACKUPDIR
cd $INDEXDIR
nice -n 19 tar -zcf $BACKUPDIR/$INDEXNAME.tar.gz $INDEXNAME
echo "DONE!"

# push both tar.gz and metadatas to tape
# echo -n "Saving to tape (this may take some time) ..."
# $BACKUPCMD $BACKUPDIR/$INDEXNAME.tar.gz $BACKUPTARGET.tar.gz
# $BACKUPCMD $MAPPING $BACKUPTARGET-mapping.json
# echo "DONE!"

# cleanup tmp files
rm $TMPDIR/*

backup_all.sh

#!/bin/bash
# 여러 인덱스를 일괄 백업
# https://gist.github.com/gypark/975f090eed1551141281aa5c070e9183 의 내용을 기반으로 수정

# 기본 세팅
ESURL="http://elasticsearch:9200"

CURDIR="/root/backup/elasticsearch"
TMPDIR=$CURDIR"/tmp"
INDEXDIR="/var/lib/elasticsearch/ESCluster/nodes/0/indices"
BACKUPDIR=$CURDIR"/es-backups"

set -x

for INDEXNAME in 인덱스이름1 인덱스이름2 ...
do

  MAPPING=$BACKUPDIR/$INDEXNAME-mapping.json

  mkdir -p $TMPDIR
  mkdir -p $BACKUPDIR

  echo "========== BACKUP [$INDEXNAME] =========="

  echo "Checking that index exist in ES"
  if [ `curl -sI $ESURL/$INDEXNAME | grep OK | wc -l` -eq 0 ]
  then
    echo "Index $INDEXNAME doesn't exist, nothing to backup"
    exit 0
  fi

  # 인덱스의 mapping, settings 정보 각각 수집
  echo -n "Backing up metadata of index $INDEXNAME ... "
  # mapping
  curl -XGET -o $TMPDIR/mapping $ESURL"/$INDEXNAME/_mapping?pretty=true" > /dev/null 2>&1
  sed -i '1,2d' $TMPDIR/mapping #strip the first two lines of the metadata
  # settings
  curl -XGET -o $TMPDIR/settings $ESURL"/$INDEXNAME/_settings?pretty=true" > /dev/null 2>&1
  head -n -2 $TMPDIR/settings | sed '1,2d' > $TMPDIR/settings_cut

  # 메타정보를 json 파일로 저장
  echo '{' > $MAPPING
  cat $TMPDIR/settings_cut >> $MAPPING
  echo ', "mappings": {' >> $MAPPING
  cat $TMPDIR/mapping >> $MAPPING
  echo "DONE!"

  # data 압축
  echo -n "Backing up data files of index $INDEXNAME (this may take some time) ... "
  cd $INDEXDIR
  nice -n 19 tar -zcf $BACKUPDIR/$INDEXNAME.tar.gz $INDEXNAME
  echo "DONE!"

  # 임시 디렉토리 청소
  rm $TMPDIR/*

done  # for loop

config.sh

#!/bin/bash

# We want to archive previous index, here last month index
INDEXNAME="logs-"`date --date="last month" +"%Y-%m"` # this had better match the index name in ES

INDEXDIR="/cygdrive/d/Data/cedric.vidal/apps/elasticsearch-0.20.2-node-1/data/cls-log-test/nodes/0/indices"
CURDIR="/cygdrive/d/Data/cedric.vidal/AMQ/CLS/backup-es"

# Local configuration
BACKUPCMD="cp"
BACKUPTARGET=$CURDIR"/es-tape/$INDEXNAME"

# S3 Configuration
# BACKUPCMD="/usr/local/backupTools/s3cmd --config=/usr/local/backupTools/s3cfg put"
# BACKUPTARGET="s3://backups/elasticsearch/$INDEXNAME"

BACKUPDIR=$CURDIR"/es-backups"

ESURL="http://localhost:9200"
TMPDIR=$CURDIR"/tmp"
MAPPING=$BACKUPDIR/$INDEXNAME-mapping.json
RESTARTCMD="" #/etc/init.d/es restart"

delete-index.sh

#!/bin/bash

. ./config.sh

echo "Checking that index exist in ES"
if [ `curl -sI $ESURL/$INDEXNAME | grep OK | wc -l` -eq 0 ]
then
  echo "Index $INDEXNAME doesn't exist, nothing to delete"
  exit 0
fi

echo -n "Deleting index $INDEXNAME ... "

curl -XDELETE "$ESURL/$INDEXNAME/"> /dev/null 2>&1
echo "DONE!"

delete.sh

#!/bin/bash

. ./config.sh

TIMESTAMPFAIL=`curl -s $ESURL/_status?pretty=true |grep index |grep log |sort |uniq |awk -F\" '{print $4}' |grep 1970 |wc -l`
if [ -n $TIMESTAMPFAIL ]
then
curl -s $ESURL/_status?pretty=true |grep index |grep log |sort |uniq |awk -F\" '{print $4}' |grep 1970 | while read line
do
echo "Indices with screwed-up timestamps found; removing"
echo -n "Deleting index $line: "
curl -s -XDELETE $ESURL/$line/
echo "DONE!"
done
fi

echo "Checking that index exist in ES"
if [ `curl -sI $ESURL/$INDEXNAME | grep OK | wc -l` -eq 0 ]
then
  echo "Index $INDEXNAME doesn't exist, nothing to delete"
  exit 0
fi

echo -n "Deleting index $INDEXNAME ... "

curl -XDELETE "$ESURL/$INDEXNAME/"> /dev/null 2>&1
echo "DONE!"

restore.sh

#!/bin/bash
# Performs 'rotation' of ES indices. Maintains only 8 indicies (1 week) of logstash logs; this script
# is to be run at midnight daily and removes the oldest one (as well as any 1970s-era log indices,
# as these are a product of timestamp fail). Please note the insane amount of error-checking
# in this script, as ES would rather delete everything than nothing…
# Before we do anything, let's get rid of any nasty 1970s-era indices we have floating around

. ./config.sh

echo "Checking that index doesn't already exist in ES"
if [ `curl -sI $ESURL/$INDEXNAME | grep OK | wc -l` -eq 1 ]
then
  echo "Index $INDEXNAME already exists, delete it before restoring. exiting"
  exit 0
fi

echo "Restoring index $INDEXNAME"

# create index and mapping
echo -n "Creating index and mappings ..."
curl -XPUT "$ESURL/$INDEXNAME/" -d @$MAPPING > /dev/null 2>&1
echo "DONE!"

# extract our data files into place
echo -n "Restoring index (this may take a while) ..."
cd $INDEXDIR
tar -zxvf $BACKUPDIR/$INDEXNAME.tar.gz
echo "DONE!"

# restart ES to allow it to open the new dir and file data
echo -n "Restarting Elasticsearch ..."
$RESTARTCMD
echo "DONE!"