lordlinus

{
  "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "metadata": {
    "_generator": {
      "name": "bicep",
      "version": "0.4.412.5873",
      "templateHash": "16564748480524566385"
    }
  },

lordlinus

pat_token_config=$(jq -n -c \
    --arg ls "$PAT_LIFETIME" \
    --arg co "Example Token" \
    '{lifetime_seconds: ($ls|tonumber),
                    comment: $co
                    }')

# Databricks Auth headers
adbGlobalToken=$(az account get-access-token --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d --output json | jq -r .accessToken)
azureApiToken=$(az account get-access-token --resource https://management.core.windows.net/ --output json | jq -r .accessToken)

lordlinus

#/bin/bash -e

az config set extension.use_dynamic_install=yes_without_prompt

# az extension add -n azure-cli-ml -y

# Attach the AKS to Azure ML as compute cluster 
null=$(az ml computetarget attach aks \
--name $AKS_COMPUTE_NAME_IN_AML \
--resource-group $RESOURCE_GROUP_NAME \

lordlinus

lordlinus

lordlinus

az account list-locations --query "[?not_null(metadata.latitude)] .{RegionName:name, PairedRegion:metadata.pairedRegion[0].name}" --output json

lordlinus

import pandas as pd
import os

folder_path = "/os/folder/path"
summaryDF = pd.DataFrame()
data = pd.concat(
    [
        pd.read_excel(os.path.join(folder_path, f), sheet_name="Data")
        for f in os.listdir(folder_path)
    ],

lordlinus

import time
from queue import Queue
from threading import Thread

table_list = [
    "table1",
    "table2",
    "table3",
    "table4",
    "table5",

lordlinus

// Fix SQL queries being returned as literals
import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects}
JdbcDialects.registerDialect(new JdbcDialect() {
override def canHandle(url: String): Boolean = url.toLowerCase.startsWith("jdbc:databricks:")
override
def quoteIdentifier(column: String): String = column
})

lordlinus