Shell Only S3 Upload Script with Multipart support. Requirements: openssl, curl. Optional: gnu-parallel.

backup.sh

#!/bin/sh
# Author: daper <[email protected]>
# Description: Shell Only S3 Upload Script with Multipart support.
# Requirements: openssl curl
# Optional: gnu-parallel

# Full path to this script.
# This is needed when using gnu-parallel.
# Please set-up manually.
SCRIPT="$(pwd)/backup.sh"

# Destination bucket.
BUCKET_NAME="<BUCKET_NAME>"

# Pass AWSPROFILE as env variable to select other profile 
# than "default" from ~/.aws/credentials.
if [ -z "$AWSPROFILE" ]; then
	AWSPROFILE="default"
fi

# args: <configKey>
function findAwsConfig {
	if [ -r ~/.aws/credentials ]; then
		cat ~/.aws/credentials \
		| sed -n "/\[$AWSPROFILE\]/,\$p" \
		| sed -n "s/$1=\(.*\)/\1/p" \
		| head -n 1
	fi
}

S3KEY=$(findAwsConfig aws_access_key_id)
S3SECRET=$(findAwsConfig aws_secret_access_key)
REGION=$(findAwsConfig region)

function getDate {
	date +"%a, %d %b %Y %T %z"
}

function getTime {
	date +%s
}

# args: <string>
function awsSign {
	echo -en "$1" \
	| openssl sha1 -hmac $S3SECRET -binary \
	| base64
}

# args: <file>
function guessContentType {
	suffix=$(echo "$1" | tr "." "\n" | tail -n 1)
	case $suffix in
		xz)
			echo "application/x-xz"
		;;
		zip)
			echo "application/zip"
		;;
		bz2)
			echo "application/x-bzip2"
		;;
		gz)
			echo "application/gzip"
		;;
		*)
			echo "application/octet-stream"
		;;
	esac
}

# args: <file>
function getSum {
	md5sum "$1" | awk '{print $1}'
}

# when called for the first time it starts counting time
# when called for the second time it calculates elapsed time
# and prints with the format of HH:MM:SS
function mark_time {
	if [ -z "$start_time" ]; then
		start_time=$(getTime)
	else
		let seconds=$(getTime)-$start_time
		if [ $seconds -ge 60 ]; then
			let minutes=$seconds/60
			let seconds=$seconds%60
			if [ $minutes -ge 60 ]; then                                                         
				let hours=$minutes/60                                                        
				let minutes=$minutes%60                                                      

				time_expression="$(printf "%02d" $hours):$(printf "%02d" $minutes):$(printf "%
				02d" $seconds)"
			else
				time_expression="00:$(printf "%02d" $minutes):$(printf "%02d" $seconds)"
			fi
		else
			time_expression="00:00:$(printf "%02d" $seconds)"
		fi

		start_time=""
		echo "$time_expression"
	fi
}

# args: <localPathOnly> <fileNameOnly> <awsPathOnly>
function putS3 {
	path=$1
	file=$2
	aws_path=$3
	
	resource="/$BUCKET_NAME/$aws_path/$file"
	contentType=$(guessContentType $file)
	dateValue=$(getDate)
	stringToSign="PUT\n$sum\n$contentType\n$dateValue\n$resource"
	signature=$(awsSign "$stringToSign")

	curl --silent --progress-bar -i -X PUT -T "$path/$file" \
		-H "Content-Type: $contentType" \
		-H "Authorization: AWS $S3KEY:$signature" \
		https://$BUCKET_NAME.s3-$REGION.amazonaws.com/$aws_path/$file
}

# args: <bucketFilePath>
function startMultipart {
	file=$(echo $1 | sed 's/^\///')
	contentType=$(guessContentType $file)
	dateValue=$(getDate)
	stringToSign="POST\n\n$contentType\n$dateValue\n/$BUCKET_NAME/$1?uploads"
	signature=$(awsSign "$stringToSign")

	resp=$(curl --silent -X POST \
		-H "Host: $BUCKET_NAME.s3.amazonaws.com" \
		-H "Date: $dateValue" \
		-H "Content-Type: ${contentType}" \
		-H "Authorization: AWS $S3KEY:$signature" \
		"https://$BUCKET_NAME.s3-$REGION.amazonaws.com/$file?uploads")

	echo "$resp" | tr "\n" " " | sed -n "s/.*<UploadId>\(.*\)<\/UploadId>.*/\1/p"
}

# args: <multiPartId> <bucketFilePath>
function abortMultipart {
	multiPartId=$1
	file=$(echo $2 | sed 's/^\///')
	dateValue=$(getDate)
	stringToSign="DELETE\n\n\n$dateValue\n/$BUCKET_NAME/$file?uploadId=$multiPartId"
	signature=$(awsSign "$stringToSign")

	resp=$(curl --silent -X DELETE \
		-H "Host: $BUCKET_NAME.s3.amazonaws.com" \
		-H "Date: $dateValue" \
		-H "Authorization: AWS $S3KEY:$signature" \
		"https://$BUCKET_NAME.s3-$REGION.amazonaws.com/$file?uploadId=$multiPartId")
}

# args: <multiPartId> <bucketFilePath> <partPath> <partNumber>
function putPart {
	multiPartId=$1
	file=$(echo $2 | sed 's/^\///')
	part=$3
	part_n=$4

	resource="/$BUCKET_NAME/$file?partNumber=$part_n&uploadId=$multiPartId"
	dateValue=$(getDate)

	stringToSign="PUT\n\n\n$dateValue\n$resource"
	signature=$(awsSign "$stringToSign")

	resp=$(curl --silent --progress-bar -i -X PUT -T "$part" \
		-H "Host: $BUCKET_NAME.s3.amazonaws.com" \
		-H "Date: $dateValue" \
		-H "Authorization: AWS $S3KEY:$signature" \
		"https://$BUCKET_NAME.s3-$REGION.amazonaws.com/$file?partNumber=$part_n&uploadId=$multiPartId")

	echo "$resp" | sed -n 's/.*ETag: "\(.*\)".*/\1/p'
}

# args: <multiPartId> <bucketFilePath> <xmlPayload>
function completeMultiPart {
	multiPartId=$1
	file=$(echo $2 | sed 's/^\///')
	payload=$3

	resource="/$BUCKET_NAME/$file"
	dateValue=$(getDate)

	stringToSign="POST\n\napplication/xml\n$dateValue\n$resource?uploadId=$multiPartId"
	signature=$(awsSign "$stringToSign")

	curl --progress-bar -i -X POST -d "$payload" \
		-H "Host: $BUCKET_NAME.s3.amazonaws.com" \
		-H "Date: $dateValue" \
		-H "Content-Length: ${#payload}" \
		-H "Content-Type: application/xml" \
		-H "Authorization: AWS $S3KEY:$signature" \
		"https://$BUCKET_NAME.s3-$REGION.amazonaws.com/$file?uploadId=$multiPartId"
}

# args: <localFilePath> <bucketFilePath>
function uploadParts {
	localFile=$1
	destination=$2
	dir=$(mktemp -d)
	split -b 100m -a 3 $localFile "$dir/part."
	let parts_n=$(ls -l $dir | wc -l)-1
	let i=1
	multiPartId=$(startMultipart $destination)
	payload="<CompleteMultipartUpload>"

	if which parallel &>/dev/null; then
		ppayload=$(mktemp)
		cmds_file=$(mktemp)
		for part in $dir/part.*; do
			part_n=$i
			echo ". $SCRIPT && echo \"$part_n \$(putPart \"$multiPartId\" \"$destination\" \"$part\" \"$part_n\")\" >> $ppayload" >> $cmds_file
			let i=$i+1
		done
		parallel &>/dev/null :::: $cmds_file
		payload="$payload$(sort -nk 1 $ppayload \
			| sed -n 's/^\([0-9]*\) \(.*\)$/<Part><PartNumber>\1<\/PartNumber><ETag>\2<\/ETag><\/Part>/gp')"
	else
		for part in $dir/part.*; do
			part_n=$i
			echo -n "[i] Uploading $part_n/$parts_n... "
			mark_time
			etag=$(putPart "$multiPartId" "$destination" "$part" "$part_n")
			mark_time
			payload="$payload<Part><PartNumber>$part_n</PartNumber><ETag>$etag</ETag></Part>"
			echo "[i] ETag: $etag"
			let i=$i+1
		done
	fi

	payload="$payload</CompleteMultipartUpload>"
	if completeMultiPart $multiPartId $destination "$payload"; then
		rm -rf $dir $ppayload $cmds_file
	fi
}