shell/bash scripts for AWS S3 upload and download (signature v4)

s3-download-aws4.sh

#!/bin/sh

# cite
# https://stackoverflow.com/a/40866205

# USAGE:
# download-aws.sh <bucket> <region> <source-file> <dest-file>

set -e

# fill this in with your credentials, this could be changed to use the same approach as s3-upload-aws4.sh.
s3Key=''
# as above
s3Secret=''

file=$3
bucket=$1
host="${bucket}.s3.amazonaws.com"
resource="/${file}"
contentType="text/plain"
dateValue="`date +'%Y%m%d'`"
X_amz_date="`date --utc +'%Y%m%dT%H%M%SZ'`"
X_amz_algorithm="AWS4-HMAC-SHA256"
awsRegion=$2
awsService="s3"
X_amz_credential="$s3Key%2F$dateValue%2F$awsRegion%2F$awsService%2Faws4_request"
X_amz_credential_auth="$s3Key/$dateValue/$awsRegion/$awsService/aws4_request"

signedHeaders="host;x-amz-algorithm;x-amz-content-sha256;x-amz-credential;x-amz-date"
# this hash is created via echo -n ''|sha256sum
contentHash="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

HMAC_SHA256_asckey () {
        var=`/bin/echo -en $2 | openssl sha256 -hmac $1 -binary | xxd -p -c256`
        echo $var
}
HMAC_SHA256 () {
        var=`/bin/echo -en $2 | openssl dgst -sha256 -mac HMAC -macopt hexkey:$1 -binary | xxd -p -c256`
        echo $var
}
REQUEST () {
        canonicalRequest="GET\n$resource\n\n"\
"host:$1\n"\
"x-amz-algorithm:$X_amz_algorithm""\n"\
"x-amz-content-sha256:$contentHash""\n"\
"x-amz-credential:$X_amz_credential""\n"\
"x-amz-date:$X_amz_date""\n\n"\
"$signedHeaders\n"\
"$contentHash"
        #echo $canonicalRequest
        canonicalHash=`/bin/echo -en "$canonicalRequest" | openssl sha256 -binary | xxd -p -c256`
        stringToSign="$X_amz_algorithm\n$X_amz_date\n$dateValue/$awsRegion/s3/aws4_request\n$canonicalHash"
        #echo $stringToSign


        s1=`HMAC_SHA256_asckey "AWS4""$s3Secret" $dateValue`
        s2=`HMAC_SHA256 "$s1" "$awsRegion"`
        s3=`HMAC_SHA256 "$s2" "$awsService"`
        signingKey=`HMAC_SHA256 "$s3" "aws4_request"`
        signature=`/bin/echo -en $stringToSign | openssl dgst -sha256 -mac HMAC -macopt hexkey:$signingKey -binary | xxd -p -c256`
        #echo signature

        authorization="$X_amz_algorithm Credential=$X_amz_credential_auth,SignedHeaders=$signedHeaders,Signature=$signature"
        result=$(curl -v -H "Host: $1" -H "X-Amz-Algorithm: $X_amz_algorithm" -H "X-Amz-Content-Sha256: $contentHash" -H "X-Amz-Credential: $X_amz_credential" -H "X-Amz-Date: $X_amz_date" -H "Authorization: $authorization" https://${1}/${file} -o "$2" --write-out "%{http_code}")
        if [ $result -eq 307 ]; then
                redirecthost=`cat $2 | sed -n 's:.*<Endpoint>\(.*\)</Endpoint>.*:\1:p'`
                REQUEST "$redirecthost" "$2"
        fi
}
REQUEST "$host" "$4"

s3-upload-aws4.sh

#!/bin/sh -u

# To the extent possible under law, Viktor Szakats (vszakats.net)
# has waived all copyright and related or neighboring rights to this
# script.
# CC0 - https://creativecommons.org/publicdomain/zero/1.0/

# USAGE:
# DO NOT prefix or suffix $file with slashes. Causes unpredictable behaviour with S3 paths.

# 4 args invocation: upload uses the relative path of the $file arg.
# AWS_CONFIG_FILE='./my-aws-config' /bin/sh ./s3-upload-aws4.sh "$file" 'your-bucket' 'eu-west-2' 'STANDARD'

# 5 args invocation: upload $file to explicit path e.g. "your/explicit/path"
# AWS_CONFIG_FILE='./my-aws-config' /bin/sh ./s3-upload-aws4.sh "$file" 'your-bucket' 'eu-west-2' 'STANDARD' 'inbox'

# where the config file looks like
# [default]
# aws_access_key_id = your_id
# aws_secret_access_key = your_key

# Upload a file to Amazon AWS S3 using Signature Version 4
#
# docs:
#    https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
#
# requires:
#    curl, openssl 1.x, GNU sed, LF EOLs in this file

fileLocal="${1:-example-local-file.ext}"
bucket="${2:-example-bucket}"
region="${3:-}"
storageClass="${4:-STANDARD}"  # or 'REDUCED_REDUNDANCY'
pathRemote="${5:-}"

# TODO sort out path/file vars to ensure there are the correct / slashs. s3 is sensitive to the incorrect slashes

m_openssl() {
  if [ -f /usr/local/opt/[email protected]/bin/openssl ]; then
    /usr/local/opt/[email protected]/bin/openssl "$@"
  elif [ -f /usr/local/opt/openssl/bin/openssl ]; then
    /usr/local/opt/openssl/bin/openssl "$@"
  else
    openssl "$@"
  fi
}

m_sed() {
  if which gsed > /dev/null 2>&1; then
    gsed "$@"
  else
    sed "$@"
  fi
}

awsStringSign4() {
  kSecret="AWS4$1"
  kDate=$(printf         '%s' "$2" | m_openssl dgst -sha256 -hex -mac HMAC -macopt "key:${kSecret}"     2>/dev/null | m_sed 's/^.* //')
  kRegion=$(printf       '%s' "$3" | m_openssl dgst -sha256 -hex -mac HMAC -macopt "hexkey:${kDate}"    2>/dev/null | m_sed 's/^.* //')
  kService=$(printf      '%s' "$4" | m_openssl dgst -sha256 -hex -mac HMAC -macopt "hexkey:${kRegion}"  2>/dev/null | m_sed 's/^.* //')
  kSigning=$(printf 'aws4_request' | m_openssl dgst -sha256 -hex -mac HMAC -macopt "hexkey:${kService}" 2>/dev/null | m_sed 's/^.* //')
  signedString=$(printf  '%s' "$5" | m_openssl dgst -sha256 -hex -mac HMAC -macopt "hexkey:${kSigning}" 2>/dev/null | m_sed 's/^.* //')
  printf '%s' "${signedString}"
}

iniGet() {
  # based on: https://stackoverflow.com/questions/22550265/read-certain-key-from-certain-section-of-ini-file-sed-awk#comment34321563_22550640
  printf '%s' "$(m_sed -n -E "/\[$2\]/,/\[.*\]/{/$3/s/(.*)=[ \\t]*(.*)/\2/p}" "$1")"
}

# Initialize access keys

if [ -z "${AWS_CONFIG_FILE:-}" ]; then
  if [ -z "${AWS_ACCESS_KEY:-}" ]; then
    echo 'AWS_CONFIG_FILE or AWS_ACCESS_KEY/AWS_SECRET_KEY envvars not set.'
    exit 1
  else
    awsAccess="${AWS_ACCESS_KEY}"
    awsSecret="${AWS_SECRET_KEY}"
    awsRegion='us-east-1'
  fi
else
  awsProfile='default'

  # Read standard aws-cli configuration file
  # pointed to by the envvar AWS_CONFIG_FILE
  awsAccess="$(iniGet "${AWS_CONFIG_FILE}" "${awsProfile}" 'aws_access_key_id')"
  awsSecret="$(iniGet "${AWS_CONFIG_FILE}" "${awsProfile}" 'aws_secret_access_key')"
  awsRegion="$(iniGet "${AWS_CONFIG_FILE}" "${awsProfile}" 'region')"
fi

# Initialize defaults

if [ -z "${pathRemote}" ]; then
  fileRemote="${fileLocal}"
else
  fileRemote="${pathRemote}/$(basename ${fileLocal})"
fi

if [ -z "${region}" ]; then
  region="${awsRegion}"
fi

echo $(date --rfc-3339=ns) [$$] "Uploading" "${fileLocal}" "->" "${bucket}" "${region}" "${storageClass}"
#echo "| $(uname) | $(m_openssl version) | $(m_sed --version | head -1) |"

# Initialize helper variables

httpReq='PUT'
authType='AWS4-HMAC-SHA256'
service='s3'
baseUrl=".${service}.amazonaws.com"
dateValueS=$(date -u +'%Y%m%d')
dateValueL=$(date -u +'%Y%m%dT%H%M%SZ')
if hash file 2>/dev/null; then
  contentType="$(file -b --mime-type "${fileLocal}")"
else
  contentType='application/octet-stream'
fi

# 0. Hash the file to be uploaded

if [ -f "${fileLocal}" ]; then
  payloadHash=$(m_openssl dgst -sha256 -hex < "${fileLocal}" 2>/dev/null | m_sed 's/^.* //')
else
  echo "File not found: '${fileLocal}'"
  exit 1
fi

# 1. Create canonical request

# NOTE: order significant in ${headerList} and ${canonicalRequest}

headerList='content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption;x-amz-storage-class'

canonicalRequest="\
${httpReq}
/${fileRemote}

content-type:${contentType}
host:${bucket}${baseUrl}
x-amz-content-sha256:${payloadHash}
x-amz-date:${dateValueL}
x-amz-server-side-encryption:AES256
x-amz-storage-class:${storageClass}

${headerList}
${payloadHash}"

# Hash it

canonicalRequestHash=$(printf '%s' "${canonicalRequest}" | m_openssl dgst -sha256 -hex 2>/dev/null | m_sed 's/^.* //')

# 2. Create string to sign

stringToSign="\
${authType}
${dateValueL}
${dateValueS}/${region}/${service}/aws4_request
${canonicalRequestHash}"

# 3. Sign the string

signature=$(awsStringSign4 "${awsSecret}" "${dateValueS}" "${region}" "${service}" "${stringToSign}")

# Upload

curl --silent --show-error --location \
  --proto-redir =https -X "${httpReq}" -T "${fileLocal}" \
  -H "Content-Type: ${contentType}" \
  -H "Host: ${bucket}${baseUrl}" \
  -H "X-Amz-Content-SHA256: ${payloadHash}" \
  -H "X-Amz-Date: ${dateValueL}" \
  -H "X-Amz-Server-Side-Encryption: AES256" \
  -H "X-Amz-Storage-Class: ${storageClass}" \
  -H "Authorization: ${authType} Credential=${awsAccess}/${dateValueS}/${region}/${service}/aws4_request, SignedHeaders=${headerList}, Signature=${signature}" \
  "https://${bucket}${baseUrl}/${fileRemote}" # >/dev/null #remove --silent and redirect stdout to /dev/null for progress bar

When I used to run your code with my own
Access and Secret keys. I found this error, could you help me to solve this?.

Error>SignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.