Skip to content

Instantly share code, notes, and snippets.

View lordlinus's full-sized avatar

Sunil Sattiraju lordlinus

View GitHub Profile
#/bin/bash -e
az config set extension.use_dynamic_install=yes_without_prompt
# az extension add -n azure-cli-ml -y
# Attach the AKS to Azure ML as compute cluster
null=$(az ml computetarget attach aks \
--name $AKS_COMPUTE_NAME_IN_AML \
--resource-group $RESOURCE_GROUP_NAME \
pat_token_config=$(jq -n -c \
--arg ls "$PAT_LIFETIME" \
--arg co "Example Token" \
'{lifetime_seconds: ($ls|tonumber),
comment: $co
}')
# Databricks Auth headers
adbGlobalToken=$(az account get-access-token --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d --output json | jq -r .accessToken)
azureApiToken=$(az account get-access-token --resource https://management.core.windows.net/ --output json | jq -r .accessToken)
{
"$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "16564748480524566385"
}
},
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
@lordlinus
lordlinus / generate_fake_data.py
Created June 18, 2021 04:16
Generate fake data
import cProfile
import io
import pstats
import random
import pandas as pd
from mimesis import Address, Datetime, Person
from mimesis.enums import Gender
person = Person('en')
addess = Address()
@lordlinus
lordlinus / synapse_spark_job.py
Created June 10, 2021 08:42
synapse spark job submit via REST api
#!/usr/bin/env python3
import os
import logging
from subprocess import run, PIPE
import requests
import json
import time
from datetime import datetime
now_file = f"{datetime.utcnow().strftime('%X').replace(':','-')}.log"
# Get 2 values from the output
command | jq -r '[.Key1.key2,.Key1.key2]'
# Get last element from list
command | jq .key[-1].key2
#! /usr/bin/bash
set -o errexit
set -o nounset
set -o pipefail
export ARM_SUBSCRIPTION_ID= XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
export ARM_TENANT_ID= XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
export ARM_CLIENT_ID=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
export ARM_CLIENT_SECRET=XXXXXXXXXXXXX
# Login using service principle
@lordlinus
lordlinus / databricks_cluster_deployment.sh
Created April 5, 2021 05:02
Bash script to deploy Databricks Cluster and other dependencies
#! /usr/bin/bash
set -o errexit
set -o nounset
set -o pipefail
export ARM_SUBSCRIPTION_ID= XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
export ARM_TENANT_ID= XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
export ARM_CLIENT_ID=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
export ARM_CLIENT_SECRET=XXXXXXXXXXXXX
export MANAGEMENT_RESOURCE_ENDPOINT="https://management.core.windows.net/" # This is Fixed value (DO NOT CHANGE)
@lordlinus
lordlinus / databricks_cluster_deployment.yml
Created April 5, 2021 04:59
Azure devops yaml pipeline to deploy Azure Databricks cluster ( end to end )
# Azure DevOps pipeline to build Databricks cluster
# This pipeline depends on "variable-group-01" to provide the below variables
# ARM_CLIENT_ID
# ARM_SUBSCRIPTION_ID
# ARM_TENANT_ID
# mySecret NOTE: This variable is mapped to ARM_CLIENT_SECRET below
trigger:
- starter