create_hdi_cluster.sh

#! /usr/bin/bash
set -o errexit
set -o nounset
set -o pipefail

export ARM_SUBSCRIPTION_ID= XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
export ARM_TENANT_ID= XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
export ARM_CLIENT_ID=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
export ARM_CLIENT_SECRET=XXXXXXXXXXXXX
# Login using service principle
echo "Logging in using Azure service priciple"
az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID
az account set -s  $ARM_SUBSCRIPTION_ID

# HDI List Cluster
# az hdinsight list

export RESOURCE_GROUP="rg-test-01"
export LOCATION="southeastasia"
export HDI_CLUSTER_NAME="demo-cluster-02"
export HDI_CLUSTER_TYPE="spark"
export HDI_COMPONENT_VERSION="Spark=3.0"
export HDI_CLUSTER_VERSION=4.0
export AZURE_STORAGE_ACCOUNT="hdistorage01"
export AZURE_STORAGE_CONTAINER="hdistoragecontainer01"

export userName="username01"
# export httpCredential=`openssl rand -base64 12`
# export sshCredentials=`openssl rand -base64 12`
export httpCredential='somesecret01#'
export sshCredentials='anothersecret02#'
export clusterSizeInNodes=3

# Create Resource Group if not exists
# NOTE: you can get list of az location from "az account list-locations | jq .[].name"
if [[ $(az group exists --resource-group $RESOURCE_GROUP) = "false" ]]; then
    echo "Resource Group does not exists, so creating.."
    az group create --name $RESOURCE_GROUP --location $LOCATION
fi

# Create storage account
# Note: kind BlobStorage is not available as the default storage account.
# az storage account create \
#     --name $AZURE_STORAGE_ACCOUNT \
#     --resource-group $RESOURCE_GROUP \
#     --https-only true \
#     --kind StorageV2 \
#     --location $LOCATION \
#     --sku Standard_LRS

export AZURE_STORAGE_KEY=$(az storage account keys list \
    --account-name $AZURE_STORAGE_ACCOUNT \
    --resource-group $RESOURCE_GROUP \
    --query [0].value -o tsv)

# Delete HDI Cluster
az hdinsight delete \
    --name $HDI_CLUSTER_NAME \
    --resource-group $RESOURCE_GROUP \
    --yes

# Create HDI Cluster
az hdinsight create \
    --name $HDI_CLUSTER_NAME \
    --resource-group $RESOURCE_GROUP \
    --type $HDI_CLUSTER_TYPE \
    --component-version $HDI_COMPONENT_VERSION \
    --http-password $httpCredential \
    --http-user $userName \
    --location $LOCATION \
    --workernode-count $clusterSizeInNodes \
    --ssh-password $sshCredentials \
    --ssh-user $userName \
    --storage-account $AZURE_STORAGE_ACCOUNT \
    --storage-account-key $AZURE_STORAGE_KEY \
    --storage-container $AZURE_STORAGE_CONTAINER \
    --timezone "Korea Standard Time" \
    --version $HDI_CLUSTER_VERSION \