Archives all cloudwatch logs to S3 for the specified environment

archive-cloudwatch-logs-to-s3.sh

#!/bin/bash -x
#
# archive-cloudwatch-logs-to-s3.sh
#
# This script archives all cloudwatch logs to S3 for the specified environment

# Environment (prefix used in all matching cloudwatch log groups - ex: prod-tomcat, prod-apache, prod-mysql, ...)
ENV=$1
# Destination S3 Bucket 
S3_LOG_BUCKET="$2"

# Get current date
CUR_DATE=$(date +%m/%Y)

# Get date or use current date if not specified
if [ ${DATE} == CUR_DATE ]
then
  DATE=${CUR_DATE}
else
  # Validate date
  if [[ ${DATE} =~ ^[0-9][0-9]/20[0-9][0-9]$ ]]
  then
    echo "${DATE} is a valid date"
  else
    echo "${DATE} is not a valid date"
    exit
  fi
fi

# Parse date
MONTH=$(echo ${DATE} | awk -F/ '{print $1}')
YEAR=$(echo ${DATE} | awk -F/ '{print $2}')

# Get dates in ms since the 1/1/1970
#===================================
# Get first day of the month
FIRST_DAY=$(date --date "${YEAR}/${MONTH}/01" +%s)000
# Get last day of the month
LAST_DAY=$(date --date "${MONTH}/1 + 1 month - 1 day" +%s)000

# Get log groups list
GROUPS_LIST=$(aws logs describe-log-groups --log-group-name-prefix ${ENV} --output text | awk '{print $4}')

for group in ${GROUPS_LIST}
do
  # Create task to export cloudwatch logs to S3
  aws logs create-export-task --task-name ${group}-${MONTH}-${YEAR} --log-group-name ${group} --from ${FIRST_DAY} --to ${LAST_DAY} --destination ${S3_LOG_BUCKET} --destination-prefix ${group}/${YEAR}/${MONTH}
  while [ $? -eq 255 ]
  # We need to wait before each export task or it will fail with the following error:
  # "An error occurred (LimitExceededException) when calling the CreateExportTask operation: Resource limit exceeded."
  do
  	sleep 60
    aws logs create-export-task --task-name ${group}-${MONTH}-${YEAR} --log-group-name ${group} --from ${FIRST_DAY} --to ${LAST_DAY} --destination ${S3_LOG_BUCKET} --destination-prefix ${group}/${YEAR}/${MONTH}
  done
done