S3 Stats

s3-stats.sh

#!/usr/bin/env bash

# minimum size to get benefit from AWS S3 intelligent tiering.
size=128000

function help() {
  echo ""
  echo "Usage: s3-stats.sh (options)"
  echo ""
  echo "  Get counts and percent of files over a given size."
  echo ""
  echo "  bucket:"
  echo "    S3 bucket."
  echo "    Example: com.fireeye.helix.delta0.us-west-2.hexint07.automation"
  echo ""
  echo "  prefix:"
  echo "    S3 key prefix."
  echo "    Example: raw/hexint07sust01/2022/05/24/"
  echo ""
  echo "  size (optional) default: ${size}"
  echo "    The size threshold."
  echo ""
}

while [[ $# -gt 0 ]]; do
case $1 in
-b|--bucket)
  bucket=$2
  shift 2
  ;;
-p|--prefix)
  prefix=$2
  shift 2
  ;;
-s|--size)
  size=$2
  shift 2
  ;;
-h|--help)
  help
  exit 0
  ;;
esac
done

if [ -z "$bucket" ]; then
  printf "\nbucket required.\n";help;exit 1
fi
if [ -z "$prefix" ]; then
  printf "\nprefix required.\n";help;exit 1
fi

echo "bucket: $bucket"
echo "prefix: $prefix"
echo "size: $size"

file_count=0
byte_total=0
large_file_count=0
large_byte_total=0

for bytes in $(aws s3 ls "$bucket/$prefix" --recursive| awk '{print $3}'); do
  let "file_count+=1"
  let "byte_total+=$bytes"
  if (( bytes > size )); then
    let "large_byte_total+=$bytes"
    let "large_file_count+=1"
  fi
done

count_percent=$(echo "$large_file_count/$file_count*100"|bc -l)
byte_percent=$(echo "$large_byte_total/$byte_total*100"|bc -l)
echo "total files: $file_count. large files: $large_file_count. percent: $count_percent"
echo "total bytes: $byte_total. large bytes: $large_byte_total. percent: $byte_percent"

exit 0

buckets=$(aws s3api list-buckets| jq --raw-output '.Buckets[].Name')
for bucket in $buckets;do
  files=$(aws s3 ls $bucket|head -n100 2>/dev/null)
  if [[ -z "$myVar" ]]; then
    echo "$bucket"
  fi
done