AWS S3 - Extract all S3 buckets' configuration as well as cloudtrail data events config. Output to json per account and per bucket. Multithreaded - use nproc minus 1.

s3_get_all_buckets_configuration.sh

#!/usr/bin/env bash

# Extract all S3 buckets' configuration as well as cloudtrail data events config. Output to json per account and per bucket. Multithreaded - use nproc minus 1.

function get_OUT(){
  ALIAS=$(aws iam list-account-aliases --query AccountAliases --output text)
  ACC=$(aws sts get-caller-identity --query Account --output text)
  if [ -z $ALIAS ]; then
    echo ${ACC}
  else
    echo "${ALIAS}_${ACC}"
  fi
}

function get_bucket_stuff(){
  bucket=$1
  echo "Processing bucket: ${bucket}"
  out="${OUT}/${bucket}"
  mkdir -p "${out}"

  echo "" > ${out}/errors
  # get bucket policy
  aws s3api get-bucket-policy --bucket ${bucket} --query Policy --output text 2>> ${out}/errors | jq > ${out}/policy.json

  # get all bucket details
  # shellcheck disable=SC2129
  aws s3api get-bucket-acl --bucket ${bucket} --query Grants > ${out}/acl.json 2>> ${out}/errors
  aws s3api get-bucket-cors --bucket ${bucket} --query CORSRules > ${out}/cors.json 2>> ${out}/errors
  aws s3api get-bucket-encryption --bucket ${bucket} --query ServerSideEncryptionConfiguration > ${out}/encryption.json 2>> ${out}/errors
  aws s3api get-bucket-lifecycle-configuration --bucket ${bucket} --query Rules > ${out}/lifecycle.json 2>> ${out}/errors
  aws s3api get-bucket-location --bucket ${bucket} --query LocationConstraint > ${out}/location.json 2>> ${out}/errors
  aws s3api get-bucket-logging --bucket ${bucket} --query LoggingEnabled > ${out}/logging.json 2>> ${out}/errors
  aws s3api get-bucket-notification-configuration --bucket ${bucket} --query LambdaFunctionConfigurations > ${out}/notification.json 2>> ${out}/errors
  aws s3api get-bucket-ownership-controls --bucket ${bucket} --query OwnershipControls > ${out}/ownership.json 2>> ${out}/errors
  aws s3api get-bucket-policy-status --bucket ${bucket} --query PolicyStatus > ${out}/policy-status.json 2>> ${out}/errors
  aws s3api get-public-access-block --bucket ${bucket} --query PublicAccessBlockConfiguration > ${out}/public-access.json 2>> ${out}/errors
  aws s3api get-bucket-replication --bucket ${bucket} --query ReplicationConfiguration > ${out}/replication.json 2>> ${out}/errors
  aws s3api get-bucket-request-payment --bucket ${bucket} --query Payer > ${out}/payment.json 2>> ${out}/errors
  aws s3api get-bucket-tagging --bucket ${bucket} --query TagSet > ${out}/tagging.json 2>> ${out}/errors
  aws s3api get-bucket-versioning --bucket ${bucket} --query Status > ${out}/versioning.json 2>> ${out}/errors
  aws s3api get-bucket-website --bucket ${bucket} --query WebsiteConfiguration > ${out}/website.json 2>> ${out}/errors
}

OUT=$(get_OUT)
echo "Output will be saved to: ${OUT}"

echo "Getting details for all buckets"
BUCKETS=$(aws s3api list-buckets --query 'Buckets[*].Name' --output text)

# Determine the maximum number of processes to run in parallel by checking available threads for CPU and deducting 1
nproc=$(nproc) # nproc gets number of cores/threads for CPU
max_processes=$((nproc-1)) # set maximum number of processes to run in parallel
echo "Max processes: $max_processes / $nproc"

# Counter for active background processes
active_processes=0

for bucket in $BUCKETS; do
  get_bucket_stuff $bucket & # Start in background
  
  # Increment the counter
  ((active_processes++))
  
  if [ "$active_processes" -ge $max_processes ]; then
    wait # Wait for all background processes to finish
    active_processes=0 # Reset counter
  fi
done

echo "Getting cloudtrail events config"
TRAILS=$(aws cloudtrail describe-trails --query 'trailList[].Name' --output text)
for trail in $TRAILS; do
  echo "Getting cloudtrail: ${trail}"
  aws cloudtrail get-event-selectors --trail-name ${trail} --query EventSelectors > ${OUT}/cloudtrail_${trail}.json 2>> ${OUT}/errors &
done

wait && echo "All done"

echo "Cleaning up empty files"
find . -size 0 -delete