A simple shell script using cURL and the Azure REST API to upload large files (> 100MB) to Azure Storage

AzureUploadLargeBlob.sh

#!/bin/bash

echo "\nEnter your storage account name:"
read AZ_ACCOUNT_NAME
echo "\nEnter your container name:"
read AZ_BLOB_CONTAINER
echo "\nEnter your SAS token (just hit ENTER/RETURN if not applicable):"
read AZ_SAS_TOKEN

DATE_NOW=$(date -Ru | sed 's/\+0000/GMT/')
AZ_VERSION="2018-03-28"
AZ_BLOB_URL="https://$AZ_ACCOUNT_NAME.blob.core.windows.net"
AZ_BLOB_TARGET="${AZ_BLOB_URL}/${AZ_BLOB_CONTAINER}/"

mkdir -p temp
XML='<?xml version="1.0" encoding="utf-8"?><BlockList>'

echo splitting $1 into 100MB chunks into temporary directory temp/
split -b 100000000 $1 temp/*
# put blocks
# ---------------------------------------
for i in temp/*
do
  echo uploading part $i
  ENCODED_I="$(openssl enc -base64 <<< $i)"
  BLOCK_ID_STRING="&comp=block&blockid=${ENCODED_I}"
  curl -i -X PUT -H "Content-Type: application/octet-stream" -H "x-ms-date: ${DATE_NOW}" -H "x-ms-version: ${AZ_VERSION}" -H "x-ms-blob-type: BlockBlob" --upload-file $i "${AZ_BLOB_TARGET}$1${AZ_SAS_TOKEN}${BLOCK_ID_STRING}"
  XML="${XML}<Uncommitted>${ENCODED_I}</Uncommitted>"
done
XML="${XML}</BlockList>"
LENGTH=${#XML}
echo "All blocks shut be put. Now attempting PutBlockList..."

# put block list. IMPORTANT: Content-Length must be the exact length of XML string, and XML must be passed to -d (--data) field
# ---------------------------------------
echo "Executing PUT for the following XML data..."
echo ${XML}
echo ""
BLOCK_ID_STRING="&comp=blocklist"
curl -i -X PUT -H "x-ms-date: ${DATE_NOW}" -H "x-ms-version: ${AZ_VERSION}" -H "Content-Length: ${LENGTH}" -d "${XML}" "${AZ_BLOB_TARGET}$1${AZ_SAS_TOKEN}${BLOCK_ID_STRING}"
echo ""
echo "Block List should be PUT. Removing temporary directory..."
rm -rf temp


# get block list. Use for debugging
# BLOCK_ID_STRING="&comp=blocklist&blocklisttype=uncommitted"
# curl -v -X GET -H "Content-Type: application/octet-stream" -H "x-ms-date: ${DATE_NOW}" -H "x-ms-version: ${AZ_VERSION}" -H "x-ms-blob-type: BlockBlob" "${AZ_BLOB_TARGET}$1${AZ_SAS_TOKEN}${BLOCK_ID_STRING}"

Thanks Ryan, this has been really useful!

I ran into an issue with multi-line block-id's creating invalid urls.

to fix the issue I added -A on line 25
from:
ENCODED_I="$(openssl enc -base64 <<< $i)"
to:
ENCODED_I="$(openssl enc -base64 -A<<< $i)"

Np! To be honest, I don't even remember writing this 😅