-
-
Save nashjain/6119aecd5e8919d0818773a118d05ed6 to your computer and use it in GitHub Desktop.
#!/bin/bash | |
deleteBefore=`date --date="1 week ago" +%F` | |
bucket=$1 | |
fileToDelete=$2 | |
fileName='aws_delete.json' | |
rm $fileName | |
echo "Removing all versions of $fileToDelete from $bucket" | |
versionsToDelete=`aws s3api list-object-versions --bucket "$bucket" --prefix "$fileToDelete" --query "Versions[?(LastModified<'$deleteBefore' && (contains(LastModified, 'T0') || contains(LastModified, 'T1') || contains(LastModified, 'T20') || contains(LastModified, 'T21') || contains(LastModified, 'T22')))].{Key: Key, VersionId: VersionId}"` | |
cat << EOF > $fileName | |
{"Objects":$versionsToDelete, "Quiet":true} | |
EOF | |
aws s3api delete-objects --bucket "$bucket" --delete file://$fileName | |
# s3api delete-objects can handle upto 1000 records | |
echo "Delete successsful" | |
# We leave the aws_delete.json file, in case you want to later see what happened. |
For deleting all objects with the above command, use this snippet to loop through all object list 1000 objects at a time:
#Get old version Objects
echo "Old version objects under this prefix:"
cat "all-objects-$bucket.json" | jq '.[] | select(.IsLatest | not)' | jq -s '.' > "old-objects-$bucket.json"
no_of_obj=$(cat old-objects-$bucket.json | jq 'length')
i=0
while [ $i -lt $no_of_obj ]
do
next=$((i+999))
oldversions=$(cat "old-objects-$bucket.json" | jq '.[] | {Key,VersionId}' | jq -s '.' | jq .[$i:$next])
cat << EOF > deleted-files-start-index-$i.json
{"Objects":$oldversions, "Quiet":true}
EOF
echo "Deleting records from $i - $next"
aws s3api delete-objects --bucket "$bucket" --delete file://deleted-files-start-index-$i.json
let i=i+1000
done
If the obj list is greater than 1000, the operation times out. Better to handle the objects in loop of some kind