list-blobs.sh

ssh 52.166.92.21

#
# Install jq (https://stedolan.github.io/jq/)
#
sudo apt-get -y install jq

#
# Install yq & xq (https://github.com/jeffbr13/xq)
#
sudo pip install yq


#
# Utility for URL encoding
#
urlencode() {
    old_lc_collate=$LC_COLLATE
    LC_COLLATE=C

    local length="${#1}"
    for (( i = 0; i < length; i++ )); do
        local c="${1:i:1}"
        case $c in
            [a-zA-Z0-9.~_-]) printf "$c" ;;
            *) printf '%%%02X' "'$c" ;;
        esac
    done

    LC_COLLATE=$old_lc_collate
}


# curl -s -H Metadata:true http://169.254.169.254/metadata/instance?api-version=2019-06-01 | jq ".compute.tags"

imdsAPI="2018-02-01"

json_identity="$( \
    curl --silent \
        --header Metadata:true \
        "http://169.254.169.254/metadata/identity/oauth2/token?api-version=${imdsAPI}&resource=$(urlencode "https://storage.azure.com/")" \
    | jq -r ".access_token")"

storageApi="2018-03-28"

container="foo"
storage_account="tmp1diag889"

#
# Download file
#
curl \
    --header "x-ms-version: ${storageApi}" \
    --header "x-ms-blob-type: BlockBlob" \
    --header "Authorization: Bearer ${json_identity}" \
    "https://${storage_account}.blob.core.windows.net/${container}/${file}" \
    --remote-name

#
# List Container
#
# Unfortunately, the REST API returns XML, so 'jq' alone isn't helpful, need to XPath into XML here
#
#
curl \
    --silent \
    --header "Authorization: Bearer ${json_identity}" \
    --header "x-ms-version: ${storageApi}" \
    "https://${storage_account}.blob.core.windows.net/${container}/?comp=list&restype=container" \
    | xq ".EnumerationResults.Blobs[]" \
    | jq -r ".[] | .Name"


#
# An alternative approach to process the XML response, but ...
#
# XML processing with Regexes guarantees us a place in hell, but we don't need 'pip install yq'
# See also https://stackoverflow.com/a/1732454/875427 for related prior art
#
curl \
    --silent \
    --header "Authorization: Bearer ${json_identity}" \
    --header "x-ms-version: ${storageApi}" \
    "https://${storage_account}.blob.core.windows.net/${container}/?comp=list&restype=container" \
    | sed -e 's|<Name>|\n<Name>|g' -e 's|</Name>|</Name>\n|g' \
    | egrep "^<Name>" \
    | sed -e  's|<Name>||g' -e  's|</Name>||g'


#
# Upload 10000 blobs
#
for i in {1..10000}
do
    echo $i
    curl \
        "https://${storage_account}.blob.core.windows.net/${container}/file-${i}.txt" \
        --request PUT \
        --header "Content-Type: text/plain; charset=UTF-8" \
        --header "x-ms-version: ${storageApi}" \
        --header "x-ms-blob-content-disposition: attachment; filename=\"${i}.txt\"" \
        --header "x-ms-blob-type: BlockBlob" \
        --header "Authorization: Bearer ${json_identity}" \
        --data "data ${i}"
done

#
# List all blobs
#
marker=""
while true;
do
    xml=$(
        curl \
            --silent \
            --header "Authorization: Bearer ${json_identity}" \
            --header "x-ms-version: ${storageApi}" \
            "https://${storage_account}.blob.core.windows.net/${container}/?comp=list&restype=container&marker=${marker}&maxresults=1000"
        )

    names="$(echo "${xml}" | xq ".EnumerationResults.Blobs[]" | jq -r ".[] | .Name" )"
    for name in $names
    do
        echo "File: ${name}"
    done

    marker="$(echo $xml | xq -r ".EnumerationResults.NextMarker")"
    if [ "${marker}" == "" -o "${marker}" == "null" ]
    then
        break
    fi
done