Mixpanel /export + Azure Blob Storage

manual-ETL-mp-azure.sh

#!/bin/bash

# Set the necessary variables
MIXPANEL_API_URL="https://data.mixpanel.com/api/2.0/export"
AZURE_ACCOUNT_NAME="mystorageaccount"
AZURE_CONTAINER_NAME="mycontainer"
AUTHORIZATION_HEADER="Authorization: Basic {{ base64(mixpanel_secret) }}"

# Function to download data from Mixpanel and upload to Azure
download_and_upload() {
  local date=$(date +%Y-%m-%d) # Sets the date variable to today's date
  local filename="mp_events_${date}.json"

  # Download data from Mixpanel
  if ! curl --request GET \
            --url "${MIXPANEL_API_URL}?from_date=${date}&to_date=${date}&=" \
            --header "Accept: application/json" \
            --header "${AUTHORIZATION_HEADER}" > "./${filename}"; then
    return 1 # Return with error code if curl fails
  fi

  # Upload to Azure Blob Storage
  if ! az storage blob upload --account-name "${AZURE_ACCOUNT_NAME}" \
                              --container-name "${AZURE_CONTAINER_NAME}" \
                              --name "${filename}" --type block \
                              --file "./${filename}" --auth-mode login; then
    rm "./${filename}" # Remove the file if upload fails
    return 1 # Return with error code if Azure CLI upload fails
  fi

  # Delete the file after a successful upload
  rm "./${filename}"
}

# Try to download and upload until successful
until download_and_upload; do
  echo "Retrying download and upload"
  sleep 60 # Wait for 60 seconds before retrying
done