bash script to retrieve files from aws s3 (or compatible) object store

s3_download.sh

#!/bin/bash

AWS_PROFILE='default' # profile name as defined in your aws config
AWS_BUCKET='' # name of the bucket you want to retrieve data from
AWS_OPTIONS_STRING='' # options you might need to add to the query
AWS_PAGE_SIZE=100
AWS_MAX_ITEMS=100
AWS_LIST_COMMAND="/usr/bin/aws --profile ${AWS_PROFILE} s3api list-objects --bucket=${AWS_BUCKET} --page-size ${AWS_PAGE_SIZE} --max-items=${AWS_MAX_ITEMS} $AWS_OPTIONS_STRING"
AWS_DOWNLOAD_COMMAND="/usr/bin/aws --profile ${AWS_PROFILE} s3 cp "
DOWNLOAD_PATH='./s3_restore' # path where to put the retrieved files

function downloadObject() {
	COMMAND="$AWS_DOWNLOAD_COMMAND $1 $2"
	$COMMAND &
}

function getObjectKeys() {

	if [ -z "$1" ]
	then
		RESPONSE=`$AWS_LIST_COMMAND`
	else
    RESPONSE=`$AWS_LIST_COMMAND --starting-token $NEXT_TOKEN`
	fi

	for row in $(echo "${RESPONSE}" | jq -r '.Contents[] | @base64'); do
	    
	    _jq() {
	     echo ${row} | base64 --decode | jq -r ${1}
	    }

	   OBJECT_KEY=$(_jq '.Key')

	   downloadObject s3://$AWS_BUCKET/$OBJECT_KEY $DOWNLOAD_PATH

	done

	NEXT_TOKEN=$(echo "${RESPONSE}" | jq -r '.NextToken')

	if [ -z "$NEXT_TOKEN" ]
	then
		echo "DONE"
		exit
	fi

	getObjectKeys $NEXT_TOKEN

}

getObjectKeys

This script will retrieve files from S3 iterating through pages