s3api.sh

s3api.sh

#!/bin/bash
set -euo pipefail;

s3_url="${S3_ENDPOINT_URL:-hb.vkcs.cloud}"
PROFILE="${AWS_PROFILE:-default}"
ACCESS_KEY="${AWS_ACCESS_KEY_ID:-}"
SECRET_KEY="${AWS_SECRET_ACCESS_KEY:-}"
REGION="${AWS_REGION:-us-east-1}"
INI_FILE=~/.aws/credentials

V4="${AWS_V4:-1}"

if [[ -z "${PROFILE}" ]]; then
	if [[ -z "${ACCESS_KEY}" ]]; then
		>&2 echo "AWS_ACCESS_KEY_ID env variable is not defined";
	fi;

	if [[ -z "${SECRET_KEY}" ]]; then
		>&2 echo "AWS_SECRET_ACCESS_KEY env variable is not defined";
	fi;
elif [[ -f "$INI_FILE" ]]; then
	while IFS=' = ' read -r key value
	do
		if [[ $key == \[*] ]]; then
			section=$key
		elif [[ $value ]] && [[ $section == "[${PROFILE}]" ]]; then
			if [[ $key == 'aws_access_key_id' ]]; then
				ACCESS_KEY="$value"
			elif [[ $key == 'aws_secret_access_key' ]]; then
				SECRET_KEY=$value
			fi
		fi
	done < $INI_FILE

	if [[ -z "${ACCESS_KEY}" || -z "${SECRET_KEY}" ]]; then
		if [[ "${AWS_PROFILE:-}" ]]; then
			>&2 echo "AWS_PROFILE is defined ${PROFILE} but no profile found in $HOME/.aws/credentials (or it is incomplete)";
		fi;
	fi;
fi;

function string2Sign() {
	http_method="$1";
	uri="$2";
	query="$3";
	curr_date="$4";

	content_md5="";
	content_type="";
	s2s="${http_method}\n${content_md5}\n${content_type}\n${curr_date}\n${uri}${query}";

	echo -n "${s2s}";
}

function signV2 () {
	s2s="$1";
	signature=$(echo -en "${s2s}" | openssl sha1 -hmac "${SECRET_KEY}" -binary | base64);
	echo "$signature";
}

function sha256hex () {
	echo -en "$1" | openssl dgst -sha256 | cut -d' ' -f2;
}

function sha256hmac () {
	echo -en "$2" | openssl dgst -sha256 -binary -hmac "$1";
}

function sha256hmacHex () {
	echo -en "$2" | openssl dgst -sha256 -hmac "$1" | cut -d' ' -f2 ;
}

function signV4 () {
	http_method="$1";
	uri="$2";
	query="${3/?/%3F}";
	curr_date="$4";
	host="$5";

	dat="$(date +%Y%m%d)";

	canonical_headers="host:${host}\nx-amz-content-sha256:UNSIGNED-PAYLOAD";
	signed_headers="host;x-amz-content-sha256";

	canonical_request="${http_method}\n${uri}\n${query}\n${canonical_headers}\n\n${signed_headers}\nUNSIGNED-PAYLOAD";

	s2s="AWS4-HMAC-SHA256\n${curr_date}\n${dat}/${REGION}/s3/aws4_request\n$(sha256hex "$canonical_request")";

	[ "$DEBUG_AUTH" ] && >&2 echo -ne "S2S>${s2s}<S2S\n\n";
	[ "$DEBUG_AUTH" ] && >&2 echo -ne "CanonicalRequest>${canonical_request}<CanonicalRequest\n\n";

	data_key=$(sha256hmac "AWS4${SECRET_KEY}" "$dat");
	# >&2 echo -ne "\nDK:${data_key}\n";

	data_region_key=$(sha256hmac "${data_key}" "${REGION}");
	# >&2 echo -ne "\nDRK:${data_region_key}\n";

	data_region_service_key=$(sha256hmac "${data_region_key}" "s3");
	# >&2 echo -ne "\nDRSK:${data_region_service_key}\n";

	signing_key=$(sha256hmac "${data_region_service_key}" "aws4_request");
	# >&2 echo -ne "\nSK:${signing_key}\n";

	sign=$(sha256hmacHex "$signing_key" "$s2s");
	# >&2 echo -ne "\nSIGN:${sign}\n";

	echo -n "AWS4-HMAC-SHA256 Credential=${ACCESS_KEY}/${dat}/${REGION}/s3/aws4_request,SignedHeaders=${signed_headers},Signature=${sign}";
}

function request() {
	http_method="$1";
	bucket_name="$2";
	sign_uri="$3";
	query="$4";

	if [[ ! "$V4" ]]; then # AWS V2
		curr_date="$(date -R)";
		s2s=$(string2Sign "${http_method}" "${sign_uri}" "" "$curr_date");
		sign=$(signV2 "$s2s");

		local host
		if [[ -z "${bucket_name}" ]]; then
			host="${s3_url}";
		else
			host="${bucket_name}.${s3_url}";
		fi;

		if [ -z "${ACCESS_KEY}" ]; then
			curl "$VERBOSE" -r "$RANGE" --compressed -w "$WRITE_OUT" -D "$DUMP_HEADER" -s -o "$OUTPUT" -k -X "${http_method}" \
				-H "Accept-Encoding: gzip" -H "Date: ${curr_date}" \
				"https://${host}/${query}"
		else
			curl "$VERBOSE" -r "$RANGE" --compressed -w "$WRITE_OUT" -D "$DUMP_HEADER" -s -o "$OUTPUT" -k -X "${http_method}" \
				-H "Accept-Encoding: gzip" -H "Date: ${curr_date}" \
				-H "Authorization: AWS ${ACCESS_KEY}:${sign}" \
				"https://${host}/${query}"
		fi;
	else # AWS V4
		curr_date="$(TZ=UTC date +"%Y%m%dT%H%M%SZ")";
		if [ -z "${ACCESS_KEY}" ]; then
			curl -r "$RANGE" --compressed -w "$WRITE_OUT" -D "$DUMP_HEADER" -s -o "$OUTPUT" -k -X "${http_method}" \
				-H "Accept-Encoding: gzip" \
				-H "X-Amz-Date: ${curr_date}" \
				"https://${host}/${query}"
		else
			auth=$(signV4 "${http_method}" "${sign_uri}" "${query}" "${curr_date}" "${s3_url}" "");
			curl -r "$RANGE" --compressed -w "$WRITE_OUT" -D "$DUMP_HEADER" -s -o "$OUTPUT" -k -X "${http_method}" \
				-H "Accept-Encoding: gzip" \
				-H "X-Amz-Date: ${curr_date}" \
				-H "X-Amz-Content-SHA256: UNSIGNED-PAYLOAD" \
				-H "Authorization: ${auth}" \
				"https://${s3_url}${sign_uri}?${query}"
		fi;
	fi;
}

function urlenc () {
	echo -n "$1" | jq -rR '. | @uri';
}

function ListPak (){
	bucket_name="$1";
	if [ -z "$bucket_name" ]; then
		>&2 echo "Bucket must be given for ListPak (please set --bucket)";
		exit 1;
	fi;

	request "GET" "${bucket_name}" "/${bucket_name}/" "?pak&max-keys=1000" | xmllint --format -;
}

function CreatePak (){
	fail="";
	if [ -z "$BUCKET" ]; then
		>&2 echo "Bucket must be given for CreatePak (please set --bucket)";
		fail=1;
	fi;
	if [ -z "$PREFIX" ]; then
		>&2 echo "Prefix must be given for CreatePak (please set --prefix)";
		fail=1;
	fi;
	if [ -z "$USERNAME" ]; then
		>&2 echo "Username must be given for CreatePak (please set --username)";
		fail=1;
	fi;

	if [[ "$fail" ]]; then
		exit 1;
	fi;

	request "PUT" "${BUCKET}" "/${BUCKET}/" "?pak&username=${USERNAME}&prefix=${PREFIX}" | xmllint --format -;
}

function DeletePak (){
	fail="";
	if [ -z "$BUCKET" ]; then
		>&2 echo "Bucket must be given for DeletePak (please set --bucket)";
		fail=1;
	fi;
	if [ -z "$PREFIX" ]; then
		>&2 echo "Prefix must be given for DeletePak (please set --prefix)";
		fail=1;
	fi;
	if [ -z "$USERNAME" ]; then
		>&2 echo "Username must be given for DeletePak (please set --username)";
		fail=1;
	fi;

	if [[ "$fail" ]]; then
		exit 1;
	fi;

	request "DELETE" "${BUCKET}" "/${BUCKET}/" "?pak&prefix=${PREFIX}&username=${USERNAME}" | xmllint --format -;
}

function ListBuckets() {
	request "GET" "" "/" "" | xmllint --format -;
}

function ListObjects() {
	bucket_name="$1";
	if [ -z "$bucket_name" ]; then
		>&2 echo "Bucket must be given for ListObjects (please set --bucket)";
		exit 1;
	fi;

	query="";
	if [[ "$V2" ]]; then
		query="list-type=2"
	fi;
	if [[ "$TOKEN" ]]; then
		query="${query}&continuation-token=$(urlenc "${TOKEN}")";
	fi;
	if [[ "${DELIMITER}" ]]; then
		query="delimiter=$(urlenc "${DELIMITER}")";
	fi;
	if [[ ! "$V2" && "$MARKER" ]]; then
		query="${query}&marker=$MARKER";
	fi;
	if [[ "${MAX_KEYS}" ]]; then
		query="${query}&max-keys=${MAX_KEYS}";
	fi;
	if [[ "${PREFIX}" ]]; then
		query="${query}&prefix=${PREFIX}";
	fi;
	if [[ "$V2" && "$MARKER" ]]; then
		query="${query}&start-after=$MARKER";
	fi;

	if (( "${#query}" > 0 )); then
		query="?${query}"
	fi;

	# method, bucket, sign_uri, uri-query
	request "GET" "${bucket_name}" "/${bucket_name}/" "${query}" | xmllint --format -;
}

function GetObject() {
	if [ -z "$BUCKET" ]; then
		>&2 echo "Bucket must be given for GetObject (please set --bucket)";
		exit 1;
	fi;
	if [ -z "$OBJECT" ]; then
		>&2 echo "Object must be given for GetObject (please set --object)";
		exit 1;
	fi;

	uri_path="/${BUCKET}/${OBJECT}";
	query="$uri_path";

	filename=$(basename "$OBJECT");
	OUTPUT="$filename";

	# method, bucket, sign_uri, uri-query
	request "GET" "${BUCKET}" "$uri_path" "${query}";
}

function DeleteBucket () {
	if [ -z "$BUCKET" ]; then
		>&2 echo "Bucket must be given for DeleteBucket (please set --bucket)";
		exit 1;
	fi;


	result=$(request "DELETE" "${BUCKET}" "/${BUCKET}/" "");
	if [ "$result" != "" ]; then
		echo "$result" | xmllint --format -;
	fi;
}

function DeleteObject () {
	if [ -z "$BUCKET" ]; then
		>&2 echo "Bucket must be given for DeleteObject (please set --bucket)";
		exit 1;
	fi;
	if [ -z "$OBJECT" ]; then
		>&2 echo "Object must be given for DeleteObject (please set --object)";
		exit 1;
	fi;

	uri_path="/${BUCKET}/${OBJECT}";
	query="";

	result=$(request "DELETE" "${BUCKET}" "$uri_path" "${query}");
	if [ "$result" != "" ]; then
		echo "$result" | xmllint --format -;
	fi;
}

POSITIONAL_ARGS=();
BUCKET="";
DELIMITER="";
PREFIX="";
USERNAME="";
V2="";
TOKEN="";
MARKER="";
DUMP_HEADER="/dev/null";
WRITE_OUT="\n";
MAX_KEYS="";
OUTPUT="-";
VERBOSE="";
OBJECT="";
RANGE="";
DEBUG_AUTH="";

while [[ $# -gt 0 ]]; do
  case $1 in
	--bucket)
	  BUCKET="$2"
	  shift # past argument
	  shift # past value
	  ;;
	--delimiter)
	  DELIMITER="$2";
	  shift # past argument
	  shift # past value
	  ;;
	--username)
	  USERNAME="$2";
	  shift # past argument
	  shift # past value
	  ;;
	--key|--object)
	  OBJECT="$2";
	  shift # past argument
	  shift # past value
	  ;;
	--range)
	  RANGE="$2";
	  shift # past argument
	  shift # past value
	  ;;
	--dump)
	#   VERBOSE="-v"
	  DUMP_HEADER="/dev/stderr";
	  WRITE_OUT="%{stderr}RemoteIP: %{remote_ip}\nDNSResolveIN: %{time_namelookup}\nTCPHandshakeIN: %{time_connect}\nSSLHandshakeIN: %{time_appconnect}\nTTFB: %{time_starttransfer}\nTotalTime: %{time_total}\nResponseBodySize: %{size_download}\nDownloadSpeed: %{speed_download}\nUploadSpeed: %{speed_upload}\n"
	  shift
	  ;;
	--debug-auth)
	  VERBOSE="-v"
	  DEBUG_AUTH="1";
	  shift
	  ;;
	--prefix)
	  PREFIX="$2";
	  shift # past argument
	  shift # past value
	  ;;
	--marker|--after)
	  MARKER="$2";
	  shift # past argument
	  shift # past value
	  ;;
	--max-keys)
	  MAX_KEYS="$2";
	  shift # past argument
	  shift # past value
	  ;;
	--v2)
	  V2="1";
	  shift # past argument
	  ;;
	--continue)
	  TOKEN="$2";
	  V2="1";
	  shift # past argument
	  shift # past value
	  ;;
   --help|-h)
		>&2 echo "Help won't come";
		exit 1;
	  ;;
	-*)
	  >&2 echo "Unknown option $1"
	  exit 1
	  ;;
	*)
	  POSITIONAL_ARGS+=("$1") # save positional arg
	  shift # past argument
	  ;;
  esac
done

if (( "${#POSITIONAL_ARGS[@]}" == 0 )); then
	>&2 echo "No command given";
	exit 1;
fi;

set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
command="$1";

if [[ -z "${command}" ]]; then
	>&2 echo "command is required";
	exit 1;
fi;

case "${command}" in
	listPak)
		ListPak "$BUCKET";
	;;
	createPak)
		CreatePak;
	;;
	deletePak)
		DeletePak;
	;;
	listBuckets)
		ListBuckets;
	;;
	deleteBucket)
		DeleteBucket;
	;;
	listObjects)
		ListObjects "$BUCKET";
	;;
	getObject)
		GetObject;
	;;
	deleteObject)
		DeleteObject;
	;;
	*)
		>&2 echo "Sorry command ${command} is not supported";
		exit 1;
	;;
esac