tailflowlogs.sh

#!/bin/bash
logpath=${HOME}/logs
random=$(cat /dev/urandom | tr -cd 'a-f0-9' | head -c 8)
logname=flowlog

mv ${logpath}/${logname} ${logpath}/archive/${logname}_${random}
mv ${logpath}/${logname}_sort ${logpath}/archive/${logname}_${random}_sort
mv ${logpath}/${logname}_top ${logpath}/archive/${logname}_${random}_top
mv ${logpath}/${logname}_reject ${logpath}/archive/${logname}_${random}_reject
mv ${logpath}/${logname}_rejectTop ${logpath}/archive/${logname}_${random}_rejectTop

# set the proxy if not
if [[ -z "${http_proxy}" ]]; then setproxy; fi

AWS_ACCOUNT=$(aws sts get-caller-identity --output text --query 'Account')
FULLPATH="s3://logbucket/AWSLogs/$AWS_ACCOUNT/vpcflowlogs/"
BUCKET=$(echo "$FULLPATH" | cut -d "/" -f3)
echo "#$BUCKET#";
echo ;

LASTFILES=$(aws s3 ls $FULLPATH --recursive --profile security | sort | tail -n 50 | awk '{print $4}')

for i in $LASTFILES; do
  if [ ! -f ${logpath}/$i ]; then
    aws s3 cp s3://$BUCKET/$i ${logpath}/$i --profile security
  fi
  gunzip -c ${logpath}/$i >> ${logpath}/${logname}
done

grep -P '^((?!version|NODATA).)*$' ${logpath}/${logname} | sort -k 11,11 > ${logpath}/${logname}_sort
cat ${logpath}/${logname}_sort | awk -f flowlogNoTime.awk | sort | uniq -c | sort -rn > ${logpath}/${logname}_top
cat ${logpath}/${logname}_sort | grep -P 'REJECT' > ${logpath}/${logname}_reject
cat ${logpath}/${logname}_sort | grep -P 'REJECT' | awk -f flowlogNoTime.awk | sort | uniq -c | sort -rn > ${logpath}/${logname}_rejectTop

echo 'Dates'
echo '  from: ' $(date -d @$(head -n 1 ${logpath}/${logname}_sort | cut -d ' ' -f 11))
echo '    to: ' $(date -d @$(tail -n 1 ${logpath}/${logname}_sort | cut -d ' ' -f 11))

echo 'Top Talkers'
echo '-----------------------------------------------------------'
head -n 25 ${logpath}/${logname}_top | awk -f resolve.awk
echo 'Top REJECT Talkers'
echo '-----------------------------------------------------------'
head -n 20 ${logpath}/${logname}_rejectTop | awk -f resolve.awk