Fetch AWS S3 Archived (Date wise) Logs

fetch-aws-s3-archived-logs-date-wise.sh

#!/bin/bash

AWS_S3_LS='s3://<BUCKET-NAME>/<PATH-TO-THE-LOGS>'
SYS_DAY=$(date '+%d')
SYS_MONTH=$(date '+%m')
SYS_YEAR=$(date '+%Y')
DEFAULT_TIME_AFTER="00:00:00"
DEFAULT_TIME_BEFORE="23:59:59"
SORT_DATE=true
LOG_TYPE='out'
DATE_SORT_COMMAND='| sort -n'

# variable="20 line one\n21 line two\n22 line two"
# $number
# awk -v var="$variable" 'BEGIN {print var}' | awk -v number="21" '$1 > number'
# echo "-----"
# awk -v var="$variable" 'BEGIN {print var}' | awk '$1 > 20'

read -e  -p "Enter year (default $SYS_YEAR): " year
year="${year:-$SYS_YEAR}"
read -e  -p "Enter month (default $SYS_MONTH): " month
month="${month:-$SYS_MONTH}"
read -e  -p "Enter date (default $SYS_DAY): " date
date="${date:-$SYS_DAY}"
read -e  -p "Sort by date (default $SORT_DATE): " sortDate
sortDate="${sortDate:-$SORT_DATE}"
read -e  -p "Log Type (default $LOG_TYPE): " logType
logType="${logType:-$LOG_TYPE}"
read -e  -p "Time before(default $DEFAULT_TIME_BEFORE): " timeBefore
timeBefore="${timeBefore:-$DEFAULT_TIME_BEFORE}"
read -e  -p "Time after(default $DEFAULT_TIME_AFTER): " timeAfter
timeAfter="${timeAfter:-$DEFAULT_TIME_AFTER}"

AWS_S3_LS="${AWS_S3_LS}/${year}/${month}/${date}/"

ALLOW_FILE_NAME="${year}${month}${date}"

echo "AWS_S3_LS --> $AWS_S3_LS"
aws s3 ls $AWS_S3_LS \
    | awk -v pattern=".*-$logType.*" '$0 ~ pattern' `# filter error or output log file` \
    | awk -v pattern=".*-$ALLOW_FILE_NAME.*" '$0 ~ pattern' `# filter error or output log file` \
    | awk -v ts="$timeAfter" '$2 > ts' `# filter time after` \
    | awk -v ts="$timeBefore" '$2 < ts' `# filter time before` \
    | sort -n `# sort timestamp descending` \
    | awk '{print $1,$2,$4}' `# print timestamp and log file names`
    # | awk '{printf "%s", $4}'
# echo "${FILES}"
echo "aws s3 cp $AWS_S3_LS"
echo "aws s3 cp $AWS_S3_LS"
echo "gunzip <your-gz-log-file>"
echo "tac <your-gz-log-file> | grep -m1 time"
echo "cat <your-gz-log-file> | grep -m1 time"
echo "cat <your-gz-log-file> | grep addDocs | grep -m6 8238"