Snowplow ES backup sync to S3

crontab

# m h  dom mon dow   command
0 * * * * /usr/sbin/es-dump
30 * * * * /usr/sbin/es-dump-s3-sync
45 0 * * * find /elasticsearch/backup/ -mtime +7 -delete

es-dump

#!/bin/bash

DUMP_FILE=/elasticsearch/backup/`date +%s`

elasticdump \
  --input=http://localhost:9200/good \
  --output=$ \
  | gzip > $DUMP_FILE.json.gz

es-dump-s3-sync

#!/bin/bash

IAM_PROFILE=`wget -q -O - http://169.254.169.254/latest/meta-data/iam/info | jq -r .InstanceProfileArn | cut -d '/' -f 2`
echo $IAM_PROFILE

case $IAM_PROFILE in
"staging-snowplow-elasticsearch")
  S3_BUCKET=staging-backups-bucket
  ;;
"production-snowplow-elasticsearch")
  S3_BUCKET=production-backups-bucket
  ;;
*)
  echo "please assign one of the following IAM roles to back up es dumps to s3\nstaging-snowplow-elasticsearch\nproduction-snowplow-elasticsearch"
  echo "staging-snowplow-elasticsearch"
  echo "production-snowplow-elasticsearch"
  exit 1
  ;;
esac

echo $S3_BUCKET
aws s3 sync /elasticsearch/backup/ s3://$S3_BUCKET