Executes AWS Inspector run, export full findings csv file from last completed run, compile a concise counters report including severity and package aggregates by hostname. Full and aggregated report are uploaded to S3.

aws_inspector_cron.sh

#!/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin:/snap/bin
log="/var/log/aws_inspector/aws_inspector_export_rep.log"
template_arn='arn:aws:inspector:us-east-1:XXXXXXXXXXXX:target/xxxxxxxxxx/template/xxxxxxxxxx'
wait_sec='5400'

log_out(){
        (($verifymon)) &&\
                echo -e "`date +'%Y-%m-%d %H:%M:%S'` (pid $$) -- $1" >> $log ||\
                echo -e "`date +'%Y-%m-%d %H:%M:%S'` (pid $$) -- $1" | tee -a $log
}

log_pipe(){
        [ ! -z $1 ] && p="$1: " || p=""
        while read data; do
                (($verifymon)) &&\
                        echo -e "`date +'%Y-%m-%d %H:%M:%S'` (pid $$) --      ${p}$data" >> $log ||\
                        echo -e "`date +'%Y-%m-%d %H:%M:%S'` (pid $$) --      ${p}$data" | tee -a $log
        done
}

log_out "=== Starting 'Full' run"
aws inspector start-assessment-run --assessment-template-arn $template_arn | log_pipe "start-assessment-run"
sleep $wait_sec

log_out "=== Compiling report"
aws_inspector_export_rep.sh

aws_inspector_export_rep.sh

#!/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin:/snap/bin

rep_dir="/var/log/aws_inspector"
[ ! -d $rep_dir ] && mkdir -p $rep_dir
log="$rep_dir/aws_inspector_export_rep.log"
bulk_size=30
bucket='my-bucket'
# days
local_retention=30

#. /opt/EC2ulz/EC2ulz.sh &>/dev/null
S=`date +%s`

log_out(){
        (($verifymon)) &&\
                echo -e "`date +'%Y-%m-%d %H:%M:%S'` (pid $$) -- $1" >> $log ||\
                echo -e "`date +'%Y-%m-%d %H:%M:%S'` (pid $$) -- $1" | tee -a $log
}

log_pipe(){
        [ ! -z "$1" ] && p="$1: " || p=""
        while read data; do
                (($verifymon)) &&\
                        echo -e "`date +'%Y-%m-%d %H:%M:%S'` (pid $$) --      ${p}$data" >> $log ||\
                        echo -e "`date +'%Y-%m-%d %H:%M:%S'` (pid $$) --      ${p}$data" | tee -a $log
        done
}

# get package names
pkg_sed=$(aws inspector describe-rules-packages --rules-package-arns `aws inspector list-rules-packages --output text |\
        awk '{print $2}'` | jq -r '.rulesPackages[] | "s#\(.arn)#\(.name)#g;"' | xargs)

# get ins to id
ins_sed=$(aws ec2 describe-instances --query 'Reservations[].Instances[].[Tags[?Key==`Name`].Value[] | [0], InstanceId]' --output text | awk '{print "s#"$2"#"$1"#;"}' | xargs)

# get last report arn
rep_list=`aws inspector list-assessment-runs --output text | while read a; do
                aws inspector describe-assessment-runs --assessment-run-arns $a
        done |\
                jq -r '.assessmentRuns[] | "\(.arn),\(.state),\(.startedAt),\(.durationInSeconds),\(.name)"' |\
                sed -r 's# ##g;' | grep 'COMPLETED' | sort -t , -k 3r`

rep_last=`echo "$rep_list" | head -1 | sed 's#,# #g'`
#e.g - arn:aws:inspector:us-east-1:665117476877:target/0-479Dbnzk/template/0-skZjgfVJ/run/0-cFJtU5Bd COMPLETED 1520554622.748 3600 Full/2018-03-09T00:17/g7mK

read last_arn x last_start x last_name <<< "$rep_last"
last_name_clean=`echo "$last_name" | sed 's#:##g; s#/#_#g;'`
last_start_date=`date -d @"${last_start/.*/}" -Isec`

#log_out "Getting report '$last_name' ($last_arn)"
log_out "Getting report '$last_name'"
full_rep="$rep_dir/${last_name_clean}_$$.full_csv"
short_rep="$rep_dir/${last_name_clean}_$$.report"

#{"assessmentRunArns":["arn:aws:inspector:us-east-1:665117476877:target/0-479Dbnzk/template/0-skZjgfVJ/run/0-rL4jPOSP"]}

# get findings for last report
t=''
find_list=$(while true; do
                o=`aws inspector list-findings --max-results 1000 --output text $t`
                f=`echo "$o" | head -1`
                [[ $f =~ ^FINDINGARNS ]] && echo "$o" && break || echo "$o" | sed 1d && t="--next-token $f"
        done | grep "$last_arn")
[ ! -z "$find_list" ] && find_count=`echo "$find_list" | wc -l` || find_count=0
log_out "Findings count: $find_count"

if [ $find_count -eq 0 ]; then
        log_out "No findings found"
        exit 1
fi

[ -t 1 ] && stat=true || stat=false
$stat && echo "  Getting all findings" >&2
C=1
echo "Host,Severity,Confidence,ID,Time,Package,Title,Description" > $full_rep
echo "$find_list" |\
        egrep '^FINDINGARNS' |\
        while read x i; do
                #echo $i >&2
                # batch processing
                if [ `echo $arns | wc -w` -eq $bulk_size ]; then
                        aws inspector describe-findings --finding-arns $arns | sed 's#\\[tn]# #g'
                        arns="$i"
                elif [ $C -eq $find_count ]; then
                        aws inspector describe-findings --finding-arns $i $arns | sed 's#\\[tn]# #g'
                else
                        arns="$arns $i"
                fi
                # one by one
                ##aws inspector describe-findings --finding-arns $i | sed 's#\\[tn]# #g'
                $stat && echo -ne "\r  $((C++))/$find_count `echo $arns | wc -w`" >&2
        done | jq -r '. | .findings[] | "\(.assetAttributes.agentId)@\(.severity)@\(.confidence)@^\(.id)^@\(.updatedAt)@^\(.serviceAttributes.rulesPackageArn)^@^\(.title)^@^\(.description)^"' |\
        sed "$ins_sed" |\
        sed "$pkg_sed" |\
        sed 's#"#`#g; s#,#.#g; s#@#,#g; s#\^#"#g' >> $full_rep
        echo
        #Etul_mapper 'Eins ID,Name' 1 plain |\
        #grep "$last_arn" #> inspector_report_${last_name//:/}_$$.csv

# prep short consice report
cat $full_rep |\
        awk 'BEGIN { FPAT = "([^, ]+)|(\"[^\"]+\")" };
                NR!=1 {
                        C[$1]++
                        CC++
                        S[$1][$2]++
                        SS[$2]++
                        gsub(/"/,"",$6)
                        P[$1][$6]++
                        PP[$6]++
                }
                END{
                        print "Host Total High Medium Low Info Practice Runtime CVE CIS"
                        for (h in C){
                                print h, C[h], S[h]["High"]+0, S[h]["Medium"]+0, S[h]["Low"]+0, S[h]["Informational"]+0, P[h]["Security Best Practices"]+0, P[h]["Runtime Behavior Analysis"]+0, P[h]["Common Vulnerabilities and Exposures"]+0, P[h]["CIS Operating System Security Configuration Benchmarks"]+0
                        }
                        print "SUM", CC, SS["High"]+0, SS["Medium"]+0, SS["Low"]+0, SS["Informational"]+0, PP["Security Best Practices"]+0, PP["Runtime Behavior Analysis"]+0, PP["Common Vulnerabilities and Exposures"]+0, PP["CIS Operating System Security Configuration Benchmarks"]+0

                }' | column -t > $short_rep

if [ ! -z "$bucket" ]; then
        aws s3 cp $full_rep s3://$bucket/full/${full_rep/*\//} | strings | egrep -v '^Completed' | log_pipe "s3 upload full"
        aws s3 cp $short_rep s3://$bucket/report/${short_rep/*\//} | strings | egrep -v '^Completed' | log_pipe "s3 upload report"
fi

find /var/log/aws_inspector/ -mtime +$local_retention | log_pipe "local cleanup ($local_retention days)"

echo ""
log_out "Duration:$((`date +%s`-$S))sec"
log_out "Run name: $last_name"
log_out "Run start date: $last_start_date"
log_out "All findings: $full_rep (s3://$bucket/full/)"
log_out "Report:       $short_rep (s3://$bucket/report/)"
log_out "Report file line count: $((`cat $full_rep | wc -l`-1))"

$stat && (echo; cat $short_rep)