Last active
February 3, 2022 21:36
-
-
Save rprakashg/97c3c4744b73122bc27b648f2ef1bd93 to your computer and use it in GitHub Desktop.
bash script to create SQS messages for files added to S3 between a specific date range
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
profile={replace} | |
s3bucket={replace} | |
folder={specify} | |
queueprefix={replace} | |
purge-sqs-queue() { | |
if [ -z "$1" ]; then | |
echo "need to pass queue name prefix" | |
return | |
fi | |
echo "getting queue url with prefix $1" | |
queueUrl=$(aws sqs list-queues --queue-name-prefix $1 --profile $profile | jq -r .QueueUrls[0]) | |
if [ -z "$queueUrl" ]; then | |
echo "Couldn't find queue, check make sure queue exist" | |
else | |
echo "purging queue : $queueUrl" | |
aws sqs purge-queue --queue-url $queueUrl --profile malabs | |
fi | |
} | |
cat << EOF > message-template.json | |
{ | |
"Records": [ | |
{ | |
"s3": { | |
"object": { "key": "" } | |
} | |
} | |
] | |
} | |
EOF | |
send-message-to-sqs() { | |
endPoint=$1 | |
key=$2 | |
if [ -z $endPoint ] || [ -z $key ]; then | |
echo "Queue URL and key are required" | |
return | |
fi | |
#message=$(cat message-template.json | jq .Records[0].s3.object.key=\"$key\" | jq 'tojson') | |
cat message-template.json | jq .Records[0].s3.object.key=\"$key\" > message.json | |
#aws sqs send-message --queue-url $endPoint --message-body ${message} --delay-seconds 0 --profile $profile | |
aws sqs send-message --queue-url $endPoint --message-body file://message.json --delay-seconds 0 --profile $profile | |
} | |
purge-sqs-queue $queueprefix | |
echo "getting list of files added to s3 for last four days" | |
#query list of files in s3 for a date range and loop through each item | |
files=($(aws s3 ls "s3://$s3bucket/$folder" --recursive --profile $profile | awk -F '<' '{split($1, d, /[- ]/)} d[2] == 8 && d[3] <= 26 && d[3] >= 22' | awk '{print $4}' | sort -r -n)) | |
for i in "${files[@]}" | |
do | |
# do whatever on $i | |
echo "Key : $i" | |
echo "sending message to sqs queue" | |
send-message-to-sqs $queueUrl $i | |
done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment