Skip to content

Instantly share code, notes, and snippets.

@lordlinus

lordlinus/test-template.json

Created July 28, 2021 01:17
Show Gist options
  • Save lordlinus/d8fea1c479c7ff26e003b00f52da71ff to your computer and use it in GitHub Desktop.
Save lordlinus/d8fea1c479c7ff26e003b00f52da71ff to your computer and use it in GitHub Desktop.
Download ZIP
Raw
test-template.json
{
"$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "16564748480524566385"
}
},
"parameters": {
"userObjectId": {
"type": "string",
"metadata": {
"description": ""
}
},
"prefix": {
"type": "string",
"defaultValue": "[uniqueString(guid(subscription().subscriptionId))]",
"metadata": {
"description": "Auto generate prefix based on subscription"
}
},
"hubVnetName": {
"type": "string",
"metadata": {
"description": ""
}
},
"spokeVnetName": {
"type": "string",
"metadata": {
"description": ""
}
},
"location": {
"type": "string",
"metadata": {
"description": ""
}
},
"adminUsername": {
"type": "string",
"metadata": {
"description": ""
}
},
"adminPassword": {
"type": "secureString",
"metadata": {
"description": ""
}
},
"webappDestinationAddresses": {
"type": "array",
"metadata": {
"description": ""
}
},
"logBlobstorageDomains": {
"type": "array",
"metadata": {
"description": ""
}
},
"extendedInfraIp": {
"type": "array",
"metadata": {
"description": ""
}
},
"sccReplayDomain": {
"type": "array",
"metadata": {
"description": ""
}
},
"metastoreDomains": {
"type": "array",
"metadata": {
"description": ""
}
},
"eventHubEndpointDomain": {
"type": "array",
"metadata": {
"description": ""
}
},
"artifactBlobStoragePrimaryDomains": {
"type": "array",
"metadata": {
"description": ""
}
},
"SpokeVnetCidr": {
"type": "string",
"metadata": {
"description": ""
}
},
"HubVnetCidr": {
"type": "string",
"metadata": {
"description": ""
}
},
"PrivateSubnetCidr": {
"type": "string",
"metadata": {
"description": ""
}
},
"PublicSubnetCidr": {
"type": "string",
"metadata": {
"description": ""
}
},
"FirewallSubnetCidr": {
"type": "string",
"metadata": {
"description": ""
}
},
"PrivateLinkSubnetCidr": {
"type": "string",
"metadata": {
"description": ""
}
}
},
"functions": [],
"variables": {
"storageSuffix": "[environment().suffixes.storage]",
"storageAccountName": "[format('{0}stg01', substring(parameters('prefix'), 0, 10))]",
"keyVaultName": "[format('{0}kv01', substring(parameters('prefix'), 0, 4))]",
"resourceGroupName": "[format('{0}-rg', substring(parameters('prefix'), 0, 4))]",
"adbWorkspaceName": "[format('{0}AdbWksp', substring(parameters('prefix'), 0, 4))]",
"nsgName": "[format('{0}nsg', substring(parameters('prefix'), 0, 4))]",
"firewallName": "[format('{0}HubFW', substring(parameters('prefix'), 0, 4))]",
"firewallPublicIpName": "[format('{0}FWPIp', substring(parameters('prefix'), 0, 4))]",
"fwRoutingTable": "[format('{0}AdbRoutingTbl', substring(parameters('prefix'), 0, 4))]",
"clientPcName": "[format('{0}ClientPc', substring(parameters('prefix'), 0, 4))]",
"eHNameSpace": "[format('eh{0}', substring(parameters('prefix'), 0, 4))]",
"eventHubName": "[variables('eHNameSpace')]"
},
"resources": [
{
"type": "Microsoft.Resources/resourceGroups",
"apiVersion": "2021-04-01",
"name": "[variables('resourceGroupName')]",
"location": "[parameters('location')]"
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2019-10-01",
"name": "RouteTable",
"resourceGroup": "[variables('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"routeTableName": {
"value": "[variables('fwRoutingTable')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "12370782830436687818"
}
},
"parameters": {
"routeTableLocation": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Azure datacentre Location to deploy the Firewall and IP Address"
}
},
"routeTableName": {
"type": "string",
"metadata": {
"description": "Name of the Routing Table"
}
}
},
"functions": [],
"resources": [
{
"type": "Microsoft.Network/routeTables",
"apiVersion": "2020-08-01",
"name": "[parameters('routeTableName')]",
"location": "[parameters('routeTableLocation')]",
"properties": {
"disableBgpRoutePropagation": false
}
},
{
"type": "Microsoft.Network/routeTables/routes",
"apiVersion": "2020-08-01",
"name": "[format('{0}/{1}', parameters('routeTableName'), 'Firewall-Route')]",
"properties": {
"addressPrefix": "0.0.0.0/0",
"nextHopType": "VirtualAppliance",
"nextHopIpAddress": "10.0.1.4",
"hasBgpOverride": false
},
"dependsOn": [
"[resourceId('Microsoft.Network/routeTables', parameters('routeTableName'))]"
]
}
],
"outputs": {
"routeTblName": {
"type": "string",
"value": "[parameters('routeTableName')]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2019-10-01",
"name": "NetworkSecurityGroup",
"resourceGroup": "[variables('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"securityGroupName": {
"value": "[variables('nsgName')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "14580252166098597018"
}
},
"parameters": {
"securityGroupName": {
"type": "string",
"metadata": {
"description": "The name of the network security group to create."
}
},
"securityGroupLocation": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Location for all resources."
}
}
},
"functions": [],
"resources": [
{
"type": "Microsoft.Network/networkSecurityGroups",
"apiVersion": "2020-05-01",
"name": "[parameters('securityGroupName')]",
"location": "[parameters('securityGroupLocation')]",
"properties": {
"securityRules": [
{
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-worker-inbound",
"properties": {
"description": "Required for worker nodes communication within a cluster.",
"protocol": "*",
"sourcePortRange": "*",
"destinationPortRange": "*",
"sourceAddressPrefix": "VirtualNetwork",
"destinationAddressPrefix": "VirtualNetwork",
"access": "Allow",
"priority": 100,
"direction": "Inbound"
}
},
{
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-databricks-webapp",
"properties": {
"description": "Required for workers communication with Databricks Webapp.",
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "443",
"sourceAddressPrefix": "VirtualNetwork",
"destinationAddressPrefix": "AzureDatabricks",
"access": "Allow",
"priority": 100,
"direction": "Outbound"
}
},
{
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-sql",
"properties": {
"description": "Required for workers communication with Azure SQL services.",
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "3306",
"sourceAddressPrefix": "VirtualNetwork",
"destinationAddressPrefix": "Sql",
"access": "Allow",
"priority": 101,
"direction": "Outbound"
}
},
{
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-storage",
"properties": {
"description": "Required for workers communication with Azure Storage services.",
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "443",
"sourceAddressPrefix": "VirtualNetwork",
"destinationAddressPrefix": "Storage",
"access": "Allow",
"priority": 102,
"direction": "Outbound"
}
},
{
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-worker-outbound",
"properties": {
"description": "Required for worker nodes communication within a cluster.",
"protocol": "*",
"sourcePortRange": "*",
"destinationPortRange": "*",
"sourceAddressPrefix": "VirtualNetwork",
"destinationAddressPrefix": "VirtualNetwork",
"access": "Allow",
"priority": 103,
"direction": "Outbound"
}
},
{
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-eventhub",
"properties": {
"description": "Required for worker communication with Azure Eventhub services.",
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "9093",
"sourceAddressPrefix": "VirtualNetwork",
"destinationAddressPrefix": "EventHub",
"access": "Allow",
"priority": 104,
"direction": "Outbound"
}
}
]
}
}
],
"outputs": {
"nsgName": {
"type": "string",
"value": "[parameters('securityGroupName')]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2019-10-01",
"name": "HubandSpokeVNET",
"resourceGroup": "[variables('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"hubVnetName": {
"value": "[parameters('hubVnetName')]"
},
"spokeVnetName": {
"value": "[parameters('spokeVnetName')]"
},
"routeTableName": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'RouteTable'), '2019-10-01').outputs.routeTblName.value]"
},
"securityGroupName": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'NetworkSecurityGroup'), '2019-10-01').outputs.nsgName.value]"
},
"firewallSubnetCidr": {
"value": "[parameters('FirewallSubnetCidr')]"
},
"hubVnetCidr": {
"value": "[parameters('HubVnetCidr')]"
},
"spokeVnetCidr": {
"value": "[parameters('SpokeVnetCidr')]"
},
"publicSubnetCidr": {
"value": "[parameters('PublicSubnetCidr')]"
},
"privateSubnetCidr": {
"value": "[parameters('PrivateSubnetCidr')]"
},
"privatelinkSubnetCidr": {
"value": "[parameters('PrivateLinkSubnetCidr')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "627072046551706049"
}
},
"parameters": {
"securityGroupName": {
"type": "string",
"metadata": {
"description": "The name of the existing network security group to create."
}
},
"spokeVnetName": {
"type": "string",
"metadata": {
"description": "The name of the virtual network to create."
}
},
"hubVnetName": {
"type": "string",
"metadata": {
"description": "The name of the virtual network to create."
}
},
"privateSubnetName": {
"type": "string",
"defaultValue": "private-subnet",
"metadata": {
"description": "The name of the private subnet to create."
}
},
"privatelinkSubnetName": {
"type": "string",
"defaultValue": "privatelink-subnet",
"metadata": {
"description": "The name of the private subnet to create."
}
},
"publicSubnetName": {
"type": "string",
"defaultValue": "public-subnet",
"metadata": {
"description": "The name of the public subnet to create."
}
},
"firewallSubnetName": {
"type": "string",
"defaultValue": "AzureFirewallSubnet",
"metadata": {
"description": "The name of the firewall subnet to create."
}
},
"routeTableName": {
"type": "string",
"metadata": {
"description": "Name of the Routing Table"
}
},
"vnetLocation": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Location for all resources."
}
},
"spokeVnetCidr": {
"type": "string",
"metadata": {
"description": "Cidr range for the spoke vnet."
}
},
"hubVnetCidr": {
"type": "string",
"metadata": {
"description": "Cidr range for the hub vnet."
}
},
"privateSubnetCidr": {
"type": "string",
"metadata": {
"description": "Cidr range for the private subnet."
}
},
"publicSubnetCidr": {
"type": "string",
"metadata": {
"description": "Cidr range for the public subnet."
}
},
"firewallSubnetCidr": {
"type": "string",
"metadata": {
"description": "Cidr range for the firewall subnet."
}
},
"privatelinkSubnetCidr": {
"type": "string",
"metadata": {
"description": "Cidr range for the private link subnet.."
}
}
},
"functions": [],
"variables": {
"securityGroupId": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('securityGroupName'))]"
},
"resources": [
{
"type": "Microsoft.Network/virtualNetworks",
"apiVersion": "2020-08-01",
"name": "[parameters('hubVnetName')]",
"location": "[parameters('vnetLocation')]",
"properties": {
"addressSpace": {
"addressPrefixes": [
"[parameters('hubVnetCidr')]"
]
},
"subnets": [
{
"name": "[parameters('firewallSubnetName')]",
"properties": {
"addressPrefix": "[parameters('firewallSubnetCidr')]",
"privateEndpointNetworkPolicies": "Enabled",
"privateLinkServiceNetworkPolicies": "Enabled",
"serviceEndpoints": [
{
"service": "Microsoft.Storage"
}
]
}
},
{
"name": "ClientDevices",
"properties": {
"addressPrefix": "10.0.200.0/24",
"routeTable": {
"id": "[resourceId('Microsoft.Network/routeTables', parameters('routeTableName'))]"
}
}
}
],
"enableDdosProtection": false
}
},
{
"type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings",
"apiVersion": "2020-08-01",
"name": "[format('{0}/{1}', parameters('hubVnetName'), 'Peer-HubSpoke')]",
"properties": {
"allowVirtualNetworkAccess": true,
"allowForwardedTraffic": true,
"allowGatewayTransit": false,
"useRemoteGateways": false,
"remoteVirtualNetwork": {
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('spokeVnetName'))]"
}
},
"dependsOn": [
"[resourceId('Microsoft.Network/virtualNetworks', parameters('hubVnetName'))]",
"[resourceId('Microsoft.Network/virtualNetworks', parameters('spokeVnetName'))]"
]
},
{
"type": "Microsoft.Network/virtualNetworks",
"apiVersion": "2020-08-01",
"name": "[parameters('spokeVnetName')]",
"location": "[parameters('vnetLocation')]",
"properties": {
"addressSpace": {
"addressPrefixes": [
"[parameters('spokeVnetCidr')]"
]
},
"subnets": [
{
"name": "[parameters('publicSubnetName')]",
"properties": {
"addressPrefix": "[parameters('publicSubnetCidr')]",
"networkSecurityGroup": {
"id": "[variables('securityGroupId')]"
},
"routeTable": {
"id": "[resourceId('Microsoft.Network/routeTables', parameters('routeTableName'))]"
},
"delegations": [
{
"name": "databricks-del-public",
"properties": {
"serviceName": "Microsoft.Databricks/workspaces"
}
}
]
}
},
{
"name": "[parameters('privateSubnetName')]",
"properties": {
"addressPrefix": "[parameters('privateSubnetCidr')]",
"networkSecurityGroup": {
"id": "[variables('securityGroupId')]"
},
"routeTable": {
"id": "[resourceId('Microsoft.Network/routeTables', parameters('routeTableName'))]"
},
"delegations": [
{
"name": "databricks-del-private",
"properties": {
"serviceName": "Microsoft.Databricks/workspaces"
}
}
]
}
},
{
"name": "[parameters('privatelinkSubnetName')]",
"properties": {
"addressPrefix": "[parameters('privatelinkSubnetCidr')]",
"delegations": [],
"privateEndpointNetworkPolicies": "Disabled",
"privateLinkServiceNetworkPolicies": "Enabled"
}
}
],
"enableDdosProtection": false
}
},
{
"type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings",
"apiVersion": "2020-08-01",
"name": "[format('{0}/{1}', parameters('spokeVnetName'), 'Peer-SpokeHub')]",
"properties": {
"allowVirtualNetworkAccess": true,
"allowForwardedTraffic": true,
"allowGatewayTransit": false,
"useRemoteGateways": false,
"remoteVirtualNetwork": {
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('hubVnetName'))]"
}
},
"dependsOn": [
"[resourceId('Microsoft.Network/virtualNetworks', parameters('hubVnetName'))]",
"[resourceId('Microsoft.Network/virtualNetworks', parameters('spokeVnetName'))]"
]
}
],
"outputs": {
"privatelinksubnet_id": {
"type": "string",
"value": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('spokeVnetName'), parameters('privatelinkSubnetName'))]"
}
}
}
},
"dependsOn": [
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'NetworkSecurityGroup')]",
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]",
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'RouteTable')]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2019-10-01",
"name": "DatabricksWorkspace",
"resourceGroup": "[variables('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"vnetName": {
"value": "[parameters('spokeVnetName')]"
},
"adbWorkspaceSkuTier": {
"value": "premium"
},
"adbWorkspaceName": {
"value": "[variables('adbWorkspaceName')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "658456181571563297"
}
},
"parameters": {
"vnetName": {
"type": "string",
"metadata": {
"description": ""
}
},
"adbWorkspaceSkuTier": {
"type": "string",
"metadata": {
"description": ""
},
"allowedValues": [
"standard",
"premium"
]
},
"publicSubnetName": {
"type": "string",
"defaultValue": "public-subnet",
"metadata": {
"description": ""
}
},
"privateSubnetName": {
"type": "string",
"defaultValue": "private-subnet",
"metadata": {
"description": ""
}
},
"disablePublicIp": {
"type": "bool",
"defaultValue": true,
"metadata": {
"description": ""
}
},
"adbWorkspaceLocation": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": ""
}
},
"adbWorkspaceName": {
"type": "string",
"metadata": {
"description": ""
}
},
"tagValues": {
"type": "object",
"defaultValue": {},
"metadata": {
"description": ""
}
}
},
"functions": [],
"variables": {
"managedResourceGroupName": "[format('databricks-rg-{0}-{1}', parameters('adbWorkspaceName'), uniqueString(parameters('adbWorkspaceName'), resourceGroup().id))]",
"managedResourceGroupId": "[format('{0}/resourceGroups/{1}', subscription().id, variables('managedResourceGroupName'))]",
"vnetId": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
},
"resources": [
{
"type": "Microsoft.Databricks/workspaces",
"apiVersion": "2018-04-01",
"name": "[parameters('adbWorkspaceName')]",
"location": "[parameters('adbWorkspaceLocation')]",
"sku": {
"name": "[parameters('adbWorkspaceSkuTier')]"
},
"properties": {
"managedResourceGroupId": "[variables('managedResourceGroupId')]",
"parameters": {
"customVirtualNetworkId": {
"value": "[variables('vnetId')]"
},
"customPublicSubnetName": {
"value": "[parameters('publicSubnetName')]"
},
"customPrivateSubnetName": {
"value": "[parameters('privateSubnetName')]"
},
"enableNoPublicIp": {
"value": "[parameters('disablePublicIp')]"
}
}
},
"tags": "[parameters('tagValues')]"
}
],
"outputs": {
"databricks_workspace_id": {
"type": "string",
"value": "[resourceId('Microsoft.Databricks/workspaces', parameters('adbWorkspaceName'))]"
},
"databricks_workspaceUrl": {
"type": "string",
"value": "[reference(resourceId('Microsoft.Databricks/workspaces', parameters('adbWorkspaceName'))).workspaceUrl]"
},
"databricks_dbfs_storage_accountName": {
"type": "string",
"value": "[reference(resourceId('Microsoft.Databricks/workspaces', parameters('adbWorkspaceName'))).parameters.storageAccountName.value]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2019-10-01",
"name": "HubFirewall",
"resourceGroup": "[variables('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"firewallName": {
"value": "[variables('firewallName')]"
},
"publicIpAddressName": {
"value": "[variables('firewallPublicIpName')]"
},
"vnetName": {
"value": "[parameters('hubVnetName')]"
},
"webappDestinationAddresses": {
"value": "[parameters('webappDestinationAddresses')]"
},
"logBlobstorageDomains": {
"value": "[parameters('logBlobstorageDomains')]"
},
"infrastructureDestinationAddresses": {
"value": "[parameters('extendedInfraIp')]"
},
"sccRelayDomains": {
"value": "[parameters('sccReplayDomain')]"
},
"metastoreDomains": {
"value": "[parameters('metastoreDomains')]"
},
"eventHubEndpointDomains": {
"value": "[parameters('eventHubEndpointDomain')]"
},
"artifactBlobStoragePrimaryDomains": {
"value": "[parameters('artifactBlobStoragePrimaryDomains')]"
},
"dbfsBlobStrageDomain": {
"value": "[array(format('{0}.blob.{1}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'DatabricksWorkspace'), '2019-10-01').outputs.databricks_dbfs_storage_accountName.value, variables('storageSuffix')))]"
},
"clientPrivateIpAddr": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'ClientPC'), '2019-10-01').outputs.clientPrivateIpaddr.value]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "11599095054178348132"
}
},
"parameters": {
"firewalllocation": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Azure datacentre Location to deploy the Firewall and IP Address"
}
},
"publicIpAddressName": {
"type": "string",
"metadata": {
"description": "Name of the IP Address"
}
},
"firewallName": {
"type": "string",
"metadata": {
"description": "Name of the Azure Firewall"
}
},
"firewallSKU": {
"type": "string",
"defaultValue": "Standard",
"metadata": {
"description": "Firewall SKU"
}
},
"clientPrivateIpAddr": {
"type": "string",
"metadata": {
"description": "Client Private ip address"
}
},
"vnetName": {
"type": "string",
"metadata": {
"description": "Name of the vnet associated witht he Firewall"
}
},
"webappDestinationAddresses": {
"type": "array",
"defaultValue": [],
"metadata": {
"description": "List of destination IP addresses for Web App"
}
},
"logBlobstorageDomains": {
"type": "array",
"defaultValue": [],
"metadata": {
"description": "List of Log Blob storage domain name"
}
},
"infrastructureDestinationAddresses": {
"type": "array",
"defaultValue": [],
"metadata": {
"description": "List of destination IP addresses for Extended Infrastructure"
}
},
"sccRelayDomains": {
"type": "array",
"defaultValue": [],
"metadata": {
"description": "List of SCC relay domain name"
}
},
"metastoreDomains": {
"type": "array",
"defaultValue": [],
"metadata": {
"description": "List of Metastore domain name"
}
},
"eventHubEndpointDomains": {
"type": "array",
"defaultValue": [],
"metadata": {
"description": "List of Event Hub endpoint domain name"
}
},
"artifactBlobStoragePrimaryDomains": {
"type": "array",
"defaultValue": [],
"metadata": {
"description": "List of Artifact Blob storage primary domain name"
}
},
"dbfsBlobStrageDomain": {
"type": "array",
"defaultValue": [],
"metadata": {
"description": "the domain name of DBFS root Blob storage"
}
}
},
"functions": [],
"variables": {
"storageSuffix": "[environment().suffixes.storage]"
},
"resources": [
{
"type": "Microsoft.Network/publicIPAddresses",
"apiVersion": "2019-02-01",
"name": "[parameters('publicIpAddressName')]",
"location": "[parameters('firewalllocation')]",
"sku": {
"name": "Standard"
},
"properties": {
"publicIPAllocationMethod": "Static"
},
"tags": {}
},
{
"type": "Microsoft.Network/azureFirewalls",
"apiVersion": "2020-05-01",
"name": "[parameters('firewallName')]",
"location": "[parameters('firewalllocation')]",
"properties": {
"ipConfigurations": [
{
"name": "[parameters('publicIpAddressName')]",
"properties": {
"subnet": {
"id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnetName'), 'AzureFirewallSubnet')]"
},
"publicIPAddress": {
"id": "[resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressName'))]"
}
}
}
],
"sku": {
"tier": "[parameters('firewallSKU')]"
},
"threatIntelMode": "Alert",
"additionalProperties": {
"Network.DNS.EnableProxy": "true"
},
"natRuleCollections": [
{
"name": "Allow-RDP-DNAT",
"properties": {
"priority": 100,
"action": {
"type": "Dnat"
},
"rules": [
{
"name": "rdp-dnat",
"protocols": [
"TCP"
],
"sourceAddresses": [
"*"
],
"destinationAddresses": [
"[reference(resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressName'))).ipAddress]"
],
"destinationPorts": [
"3389"
],
"translatedAddress": "[parameters('clientPrivateIpAddr')]",
"translatedPort": "3389"
}
]
}
}
],
"networkRuleCollections": [
{
"name": "Allow-Databricks-Services",
"properties": {
"priority": 100,
"action": {
"type": "Allow"
},
"rules": [
{
"name": "Webapp IP",
"protocols": [
"TCP"
],
"sourceAddresses": [
"*"
],
"destinationAddresses": "[parameters('webappDestinationAddresses')]",
"sourceIpGroups": [],
"destinationIpGroups": [],
"destinationFqdns": [],
"destinationPorts": [
"*"
]
},
{
"name": "Extended infrastructure IP",
"protocols": [
"TCP"
],
"sourceAddresses": [
"*"
],
"destinationAddresses": "[parameters('infrastructureDestinationAddresses')]",
"sourceIpGroups": [],
"destinationIpGroups": [],
"destinationFqdns": [],
"destinationPorts": [
"*"
]
},
{
"name": "Metastore IP",
"protocols": [
"TCP"
],
"sourceAddresses": [
"*"
],
"destinationAddresses": [],
"sourceIpGroups": [],
"destinationIpGroups": [],
"destinationFqdns": "[parameters('metastoreDomains')]",
"destinationPorts": [
"*"
]
},
{
"name": "Event Hub endpoint",
"protocols": [
"TCP"
],
"sourceAddresses": [
"*"
],
"destinationAddresses": [],
"sourceIpGroups": [],
"destinationIpGroups": [],
"destinationFqdns": "[parameters('eventHubEndpointDomains')]",
"destinationPorts": [
"*"
]
}
]
}
}
],
"applicationRuleCollections": [
{
"name": "Allow-Databricks-Services",
"properties": {
"priority": 100,
"action": {
"type": "Allow"
},
"rules": [
{
"name": "Log Blob storage IP",
"protocols": [
{
"protocolType": "Http",
"port": 80
},
{
"protocolType": "Https",
"port": 443
}
],
"fqdnTags": [],
"targetFqdns": "[parameters('logBlobstorageDomains')]",
"sourceAddresses": [
"*"
],
"sourceIpGroups": []
},
{
"name": "SCC Relay IP",
"protocols": [
{
"protocolType": "Http",
"port": 80
},
{
"protocolType": "Https",
"port": 443
}
],
"fqdnTags": [],
"targetFqdns": "[parameters('sccRelayDomains')]",
"sourceAddresses": [
"*"
],
"sourceIpGroups": []
},
{
"name": "Artifact Blob storage IP",
"protocols": [
{
"protocolType": "Http",
"port": 80
},
{
"protocolType": "Https",
"port": 443
}
],
"fqdnTags": [],
"targetFqdns": "[parameters('artifactBlobStoragePrimaryDomains')]",
"sourceAddresses": [
"*"
],
"sourceIpGroups": []
},
{
"name": "DBFS root Blob storage IP",
"protocols": [
{
"protocolType": "Http",
"port": 80
},
{
"protocolType": "Https",
"port": 443
}
],
"fqdnTags": [],
"targetFqdns": "[parameters('dbfsBlobStrageDomain')]",
"sourceAddresses": [
"*"
],
"sourceIpGroups": []
}
]
}
},
{
"name": "Allow-Websites",
"properties": {
"priority": 200,
"action": {
"type": "Allow"
},
"rules": [
{
"name": "Pypi",
"protocols": [
{
"protocolType": "Http",
"port": 80
},
{
"protocolType": "Https",
"port": 443
}
],
"fqdnTags": [],
"targetFqdns": [
"*.pypi.org",
"*.pythonhosted.org"
],
"sourceAddresses": [
"*"
],
"sourceIpGroups": []
},
{
"name": "Maven",
"protocols": [
{
"protocolType": "Http",
"port": 80
},
{
"protocolType": "Https",
"port": 443
}
],
"fqdnTags": [],
"targetFqdns": [
"*.mvnrepository.com",
"*.maven.org"
],
"sourceAddresses": [
"*"
],
"sourceIpGroups": []
},
{
"name": "GitHubScripts",
"protocols": [
{
"protocolType": "Http",
"port": 80
},
{
"protocolType": "Https",
"port": 443
}
],
"fqdnTags": [],
"targetFqdns": [
"*.githubusercontent.com",
"github.com"
],
"sourceAddresses": [
"*"
],
"sourceIpGroups": []
},
{
"name": "LogAnalytics",
"protocols": [
{
"protocolType": "Http",
"port": 80
},
{
"protocolType": "Https",
"port": 443
}
],
"fqdnTags": [],
"targetFqdns": [
"*.ods.opinsights.azure.com",
"*.oms.opinsights.azure.com",
"[format('*.blob.{0}', variables('storageSuffix'))]",
"*.azure-automation.net"
],
"sourceAddresses": [
"*"
],
"sourceIpGroups": []
}
]
}
}
]
},
"tags": {},
"dependsOn": [
"[resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressName'))]"
]
}
]
}
},
"dependsOn": [
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'DatabricksWorkspace')]",
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'ClientPC')]",
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2019-10-01",
"name": "StorageAccount",
"resourceGroup": "[variables('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"storageAccountName": {
"value": "[variables('storageAccountName')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "11740907641133745470"
}
},
"parameters": {
"storageAccountName": {
"type": "string",
"metadata": {
"description": "Name of the storage account"
},
"maxLength": 24,
"minLength": 3
},
"storageContainerName": {
"type": "string",
"defaultValue": "data"
},
"storageAccountSku": {
"type": "string",
"defaultValue": "Standard_LRS",
"metadata": {
"description": "Storage Account Sku"
},
"allowedValues": [
"Standard_LRS",
"Standard_GRS",
"Standard_ZRS",
"Premium_LRS"
]
},
"storageAccountLocation": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Location for all resources."
}
},
"encryptionEnabled": {
"type": "bool",
"defaultValue": true,
"metadata": {
"description": "Enable or disable Blob encryption at Rest."
}
}
},
"functions": [],
"resources": [
{
"type": "Microsoft.Storage/storageAccounts",
"apiVersion": "2019-06-01",
"name": "[parameters('storageAccountName')]",
"tags": {
"displayName": "[parameters('storageAccountName')]",
"type": "Storage"
},
"location": "[parameters('storageAccountLocation')]",
"kind": "StorageV2",
"properties": {
"isHnsEnabled": true,
"minimumTlsVersion": "TLS1_2",
"supportsHttpsTrafficOnly": true,
"accessTier": "Hot",
"networkAcls": {
"bypass": "AzureServices",
"virtualNetworkRules": [],
"ipRules": [],
"defaultAction": "Deny"
},
"encryption": {
"keySource": "Microsoft.Storage",
"services": {
"blob": {
"enabled": "[parameters('encryptionEnabled')]"
},
"file": {
"enabled": "[parameters('encryptionEnabled')]"
}
}
}
},
"sku": {
"name": "[parameters('storageAccountSku')]"
}
},
{
"type": "Microsoft.Storage/storageAccounts/blobServices/containers",
"apiVersion": "2019-06-01",
"name": "[format('{0}/default/{1}', parameters('storageAccountName'), parameters('storageContainerName'))]",
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]"
]
}
],
"outputs": {
"key1": {
"type": "string",
"value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-04-01').keys[0].value]"
},
"key2": {
"type": "string",
"value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-04-01').keys[1].value]"
},
"storageaccount_id": {
"type": "string",
"value": "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2019-10-01",
"name": "KeyVault",
"resourceGroup": "[variables('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"keyVaultName": {
"value": "[variables('keyVaultName')]"
},
"objectId": {
"value": "[parameters('userObjectId')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "14027008661142927497"
}
},
"parameters": {
"keyVaultName": {
"type": "string",
"metadata": {
"description": "Specifies the name of the key vault."
}
},
"keyVaultLocation": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Specifies the Azure location where the key vault should be created."
}
},
"enabledForDeployment": {
"type": "bool",
"defaultValue": false,
"metadata": {
"description": "Specifies whether Azure Virtual Machines are permitted to retrieve certificates stored as secrets from the key vault."
},
"allowedValues": [
true,
false
]
},
"enabledForTemplateDeployment": {
"type": "bool",
"defaultValue": false,
"metadata": {
"description": "Specifies whether Azure Resource Manager is permitted to retrieve secrets from the key vault."
},
"allowedValues": [
true,
false
]
},
"tenantId": {
"type": "string",
"defaultValue": "[subscription().tenantId]",
"metadata": {
"description": "Specifies the Azure Active Directory tenant ID that should be used for authenticating requests to the key vault. Get it by using Get-AzSubscription cmdlet."
}
},
"objectId": {
"type": "string",
"metadata": {
"description": "Specifies the object ID of a user, service principal or security group in the Azure Active Directory tenant for the vault. The object ID must be unique for the list of access policies. Get it by using Get-AzADUser or Get-AzADServicePrincipal cmdlets."
}
},
"keyVaultSkuTier": {
"type": "string",
"defaultValue": "standard",
"metadata": {
"description": "Specifies whether the key vault is a standard vault or a premium vault."
},
"allowedValues": [
"standard",
"premium"
]
},
"tagValues": {
"type": "object",
"defaultValue": {}
}
},
"functions": [],
"resources": [
{
"type": "Microsoft.KeyVault/vaults",
"apiVersion": "2019-09-01",
"name": "[parameters('keyVaultName')]",
"location": "[parameters('keyVaultLocation')]",
"properties": {
"enabledForDeployment": "[parameters('enabledForDeployment')]",
"enabledForTemplateDeployment": "[parameters('enabledForTemplateDeployment')]",
"tenantId": "[parameters('tenantId')]",
"accessPolicies": [
{
"objectId": "[parameters('objectId')]",
"tenantId": "[parameters('tenantId')]",
"permissions": {
"secrets": [
"list",
"get",
"set"
]
}
}
],
"sku": {
"name": "[parameters('keyVaultSkuTier')]",
"family": "A"
},
"networkAcls": {
"defaultAction": "Deny",
"bypass": "AzureServices",
"ipRules": [],
"virtualNetworkRules": []
}
},
"tags": "[parameters('tagValues')]"
}
],
"outputs": {
"keyvault_id": {
"type": "string",
"value": "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]"
},
"keyvault_uri": {
"type": "string",
"value": "[reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))).vaultUri]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2019-10-01",
"name": "ClientPC",
"resourceGroup": "[variables('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"adminUsername": {
"value": "[parameters('adminUsername')]"
},
"adminPassword": {
"value": "[parameters('adminPassword')]"
},
"vnetName": {
"value": "[parameters('hubVnetName')]"
},
"clientPcName": {
"value": "[variables('clientPcName')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "6613247302197643622"
}
},
"parameters": {
"clientPcName": {
"type": "string",
"metadata": {
"description": ""
}
},
"osDiskType": {
"type": "string",
"defaultValue": "Premium_LRS",
"metadata": {
"description": ""
}
},
"virtualMachineSize": {
"type": "string",
"defaultValue": "Standard_DS1_v2",
"metadata": {
"description": ""
}
},
"patchMode": {
"type": "string",
"defaultValue": "AutomaticByOS",
"metadata": {
"description": ""
}
},
"vnetName": {
"type": "string",
"metadata": {
"description": ""
}
},
"adminUsername": {
"type": "string",
"metadata": {
"description": ""
}
},
"location": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": ""
}
},
"adminPassword": {
"type": "secureString",
"metadata": {
"description": ""
}
}
},
"functions": [],
"variables": {
"networkInterfaceName": "[format('{0}-Iface', parameters('clientPcName'))]",
"virtualMachineName": "[parameters('clientPcName')]",
"subnetRef": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnetName'), 'ClientDevices')]"
},
"resources": [
{
"type": "Microsoft.Network/networkInterfaces",
"apiVersion": "2018-10-01",
"name": "[variables('networkInterfaceName')]",
"location": "[parameters('location')]",
"properties": {
"ipConfigurations": [
{
"name": "ipconfig1",
"properties": {
"subnet": {
"id": "[variables('subnetRef')]"
},
"privateIPAllocationMethod": "Dynamic"
}
}
]
}
},
{
"type": "Microsoft.Compute/virtualMachines",
"apiVersion": "2021-03-01",
"name": "[variables('virtualMachineName')]",
"location": "[parameters('location')]",
"properties": {
"hardwareProfile": {
"vmSize": "[parameters('virtualMachineSize')]"
},
"storageProfile": {
"osDisk": {
"createOption": "FromImage",
"managedDisk": {
"storageAccountType": "[parameters('osDiskType')]"
}
},
"imageReference": {
"publisher": "MicrosoftWindowsDesktop",
"offer": "Windows-10",
"sku": "20h2-pro-g2",
"version": "latest"
}
},
"networkProfile": {
"networkInterfaces": [
{
"id": "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]"
}
]
},
"osProfile": {
"computerName": "[variables('virtualMachineName')]",
"adminUsername": "[parameters('adminUsername')]",
"adminPassword": "[parameters('adminPassword')]",
"windowsConfiguration": {
"enableAutomaticUpdates": true,
"provisionVMAgent": true,
"patchSettings": {
"enableHotpatching": false,
"patchMode": "[parameters('patchMode')]"
}
}
},
"licenseType": "Windows_Client",
"diagnosticsProfile": {
"bootDiagnostics": {
"enabled": true
}
}
},
"dependsOn": [
"[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]"
]
}
],
"outputs": {
"clientPrivateIpaddr": {
"type": "string",
"value": "[reference(resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))).ipConfigurations[0].properties.privateIPAddress]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]",
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVNET')]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2019-10-01",
"name": "loganalytics",
"resourceGroup": "[variables('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "2031238284435439699"
}
},
"parameters": {
"logAnalyticsWkspName": {
"type": "string",
"defaultValue": "[toLower(format('spark-monitoring-{0}', uniqueString(resourceGroup().name)))]"
},
"logAnalyticsWkspLocation": {
"type": "string",
"defaultValue": "[resourceGroup().location]"
},
"logAnalyticsWkspSku": {
"type": "string",
"defaultValue": "Standalone",
"metadata": {
"description": "Service Tier: Free, Standalone, PerNode, or PerGB2018"
},
"allowedValues": [
"Free",
"Standalone",
"PerNode",
"PerGB2018"
]
},
"logAnalyticsWkspRentationDays": {
"type": "int",
"defaultValue": 30,
"metadata": {
"description": "Number of days of retention. Free plans can only have 7 days, Standalone and Log Analytics plans include 30 days for free"
},
"maxValue": 730,
"minValue": 7
}
},
"functions": [],
"variables": {
"queries": [
{
"displayName": "Stage Latency Per Stage (Stage Duration)",
"query": "let results=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,apptag,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerStageCompleted\" \n | extend stageDuration=Stage_Info_Completion_Time_d - Stage_Info_Submission_Time_d\n) on Stage_Info_Stage_ID_d;\nresults\n | extend slice = strcat(Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s) \n| extend stageDuration=Stage_Info_Completion_Time_d - Stage_Info_Submission_Time_d \n| summarize percentiles(stageDuration,10,30,50,90) by bin(TimeGenerated, 1m), slice\n| order by TimeGenerated asc nulls last\n\n"
},
{
"displayName": "Stage Throughput Per Stage",
"query": "let results=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project \nStage_Info_Stage_ID_d,apptag,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerStageCompleted\" \n) on Stage_Info_Stage_ID_d;\nresults\n | extend slice = strcat(\"# StagesCompleted \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",\napptag,\" \",Stage_Info_Stage_Name_s) \n| summarize StagesCompleted=count(Event_s) by bin(TimeGenerated,1m), slice\n| order by TimeGenerated asc nulls last\n\n"
},
{
"displayName": "Tasks Per Stage",
"query": "let results=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project \nStage_Info_Stage_ID_d,apptag,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerStageCompleted\" \n) on Stage_Info_Stage_ID_d;\nresults\n | extend slice = strcat(\"# StagesCompleted \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| extend slice=strcat(Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s) \n| project Stage_Info_Number_of_Tasks_d,slice,TimeGenerated \n| order by TimeGenerated asc nulls last\n\n"
},
{
"displayName": "% Serialize Time Per Executor",
"query": "let results = SparkMetric_CL\n| where name_s contains \"executor.resultserializationtime\" \n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , setime=count_d , executor ,name_s\n| join kind= inner (\nSparkMetric_CL\n| where name_s contains \"executor.RunTime\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , runTime=count_d , executor ,name_s\n) on executor, TimeGenerated;\nresults\n| extend serUsage=(setime/runTime)*100\n| summarize SerializationCpuTime=percentile(serUsage,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n| render timechart "
},
{
"displayName": "Shuffle Bytes Read Per Executor",
"query": "let results=SparkMetric_CL\n| where name_s contains \"executor.shuffleTotalBytesRead\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkMetric_CL\n | where name_s contains \"executor.shuffleTotalBytesRead\"\n | extend sname=split(name_s, \".\") \n | extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\n) on executor, TimeGenerated;\nresults\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \n| summarize max(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n"
},
{
"displayName": "Error Traces (Bad Record Or Bad Files)",
"query": "SparkListenerEvent_CL\r\n| where Level contains \"Error\"\r\n| project TimeGenerated , Message \r\n"
},
{
"displayName": "Task Shuffle Bytes Written",
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend schedulerdelay = Task_Info_Launch_Time_d - Stage_Info_Submission_Time_d\n| extend name=strcat(\"SchuffleBytesWritten \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
},
{
"displayName": "Task Input Bytes Read",
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,Task_Metrics_Input_Metrics_Bytes_Read_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"InputBytesRead \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Input_Metrics_Bytes_Read_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
},
{
"displayName": "Sum Task Execution Per Host",
"query": "SparkListenerEvent_CL\n| where Event_s contains \"taskend\" \n| extend taskDuration=Task_Info_Finish_Time_d-Task_Info_Launch_Time_d \n| summarize sum(taskDuration) by bin(TimeGenerated, 1m), Task_Info_Host_s\n| order by TimeGenerated asc nulls last "
},
{
"displayName": "% CPU Time Per Executor",
"query": "let results = SparkMetric_CL \n| where name_s contains \"executor.cpuTime\" \n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , cpuTime=count_d/1000000 , executor ,name_s\n| join kind= inner (\n SparkMetric_CL\n| where name_s contains \"executor.RunTime\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , runTime=count_d , executor ,name_s\n) on executor, TimeGenerated;\nresults\n| extend cpuUsage=(cpuTime/runTime)*100\n| summarize ExecutorCpuTime = percentile(cpuUsage,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last \n"
},
{
"displayName": "Job Throughput",
"query": "let results=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerJobStart\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\n| project Job_ID_d,apptag,Properties_spark_databricks_clusterUsageTags_clusterName_s,TimeGenerated\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerJobEnd\"\n | where Job_Result_Result_s contains \"JobSucceeded\"\n | project Event_s,Job_ID_d,TimeGenerated\n) on Job_ID_d;\nresults\n| extend slice=strcat(\"#JobsCompleted \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag)\n| summarize count(Event_s) by bin(TimeGenerated, 1m),slice\n| order by TimeGenerated asc nulls last"
},
{
"displayName": "Shuffle Disk Bytes Spilled Per Executor",
"query": "let results=SparkMetric_CL\r\n| where name_s contains \"executor.diskBytesSpilled\"\r\n| extend sname=split(name_s, \".\") \r\n| extend executor=strcat(\"executorid:\",sname[1])\r\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \r\n| order by TimeGenerated asc nulls last \r\n| join kind= inner (\r\n SparkMetric_CL\r\n | where name_s contains \"executor.diskBytesSpilled\"\r\n | extend sname=split(name_s, \".\") \r\n | extend executor=strcat(\"executorid:\",sname[1])\r\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\r\n) on executor, TimeGenerated;\r\nresults\r\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \r\n| summarize any(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\r\n| order by TimeGenerated asc nulls last\r\n"
},
{
"displayName": "Task Shuffle Read Time",
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"TaskShuffleReadTime \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
},
{
"displayName": "Shuffle Heap Memory Per Executor",
"query": "SparkMetric_CL\n| where name_s contains \"shuffle-client.usedHeapMemory\"\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize percentile(value_d,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last"
},
{
"displayName": "Job Errors Per Job",
"query": "let results=SparkListenerEvent_CL\r\n| where Event_s contains \"SparkListenerJobStart\"\r\n| project Job_ID_d,Properties_callSite_short_s,TimeGenerated\r\n| order by TimeGenerated asc nulls last \r\n| join kind= inner (\r\n SparkListenerEvent_CL\r\n | where Event_s contains \"SparkListenerJobEnd\"\r\n | where Job_Result_Result_s !contains \"JobSucceeded\"\r\n | project Event_s,Job_ID_d,TimeGenerated\r\n) on Job_ID_d;\r\nresults\r\n| extend slice=strcat(\"JobErrors \",Properties_callSite_short_s)\r\n| summarize count(Event_s) by bin(TimeGenerated, 1m),slice\r\n| order by TimeGenerated asc nulls last"
},
{
"displayName": "Task Errors Per Stage",
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageCompleted\"\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Event_s,TimeGenerated\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s !contains \"Success\"\n | project Stage_ID_d,Task_Info_Task_ID_d,Task_End_Reason_Reason_s,\n TaskEvent=Event_s,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend slice=strcat(\"#TaskErrors \",Stage_Info_Stage_Name_s)\n| summarize count(TaskEvent) by bin(TimeGenerated,1m),slice\n| order by TimeGenerated asc nulls last\n"
},
{
"displayName": "Streaming Latency Per Stream",
"query": "\r\n\r\nSparkListenerEvent_CL\r\n| where Event_s contains \"queryprogressevent\"\r\n| extend sname=strcat(progress_name_s,\"-\",\"triggerexecution\") \r\n| summarize percentile(progress_durationMs_triggerExecution_d,90) by bin(TimeGenerated, 1m), sname\r\n| order by TimeGenerated asc nulls last \r\n"
},
{
"displayName": "Task Shuffle Write Time",
"query": "let result=SparkListenerEvent_CL\r\n| where Event_s contains \"SparkListenerStageCompleted\"\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated\r\n| order by TimeGenerated asc nulls last \r\n| join kind= inner (\r\n SparkListenerEvent_CL\r\n | where Event_s contains \"SparkListenerTaskEnd\"\r\n | where Task_End_Reason_Reason_s contains \"Success\"\r\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\r\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\r\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\r\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\r\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\r\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\r\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\r\nresult\r\n| extend ShuffleWriteTime=Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d/1000000\r\n| extend name=strcat(\"TaskShuffleWriteTime \",Stage_Info_Stage_Name_s)\r\n| summarize percentile(ShuffleWriteTime,90) by bin(TimeGenerated,1m),name\r\n| order by TimeGenerated asc nulls last;\r\n\r\n"
},
{
"displayName": "Task Deserialization Time",
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,Task_Metrics_Input_Metrics_Bytes_Read_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"TaskDeserializationTime \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Executor_Deserialize_Time_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
},
{
"displayName": "Task Result Serialization Time",
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"TaskResultSerializationTime \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Result_Serialization_Time_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
},
{
"displayName": "File System Bytes Read Per Executor",
"query": "SparkMetric_CL\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| where name_s contains \"executor.filesystem.file.read_bytes\" \n| summarize FileSystemReadBytes=percentile(value_d,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last"
},
{
"displayName": "Streaming Throughput Processed Rows/Sec",
"query": "SparkListenerEvent_CL\r\n| where Event_s contains \"progress\"\r\n| extend sname=strcat(progress_name_s,\"-ProcRowsPerSecond\") \r\n| extend status = todouble(extractjson(\"$.[0].processedRowsPerSecond\", progress_sources_s))\r\n| summarize percentile(status,90) by bin(TimeGenerated, 1m) , sname\r\n| order by TimeGenerated asc nulls last "
},
{
"displayName": "% Deserialize Time Per Executor",
"query": "let results = SparkMetric_CL \n| where name_s contains \"executor.deserializetime\" \n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , desetime=count_d , executor ,name_s\n| join kind= inner (\nSparkMetric_CL\n| where name_s contains \"executor.RunTime\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , runTime=count_d , executor ,name_s\n) on executor, TimeGenerated;\nresults\n| extend deseUsage=(desetime/runTime)*100\n| summarize deSerializationCpuTime=percentiles(deseUsage,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last "
},
{
"displayName": "Tasks Per Executor (Sum Of Tasks Per Executor)",
"query": "SparkMetric_CL\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1]) \n| where name_s contains \"threadpool.activeTasks\" \n| summarize percentile(value_d,90) by bin(TimeGenerated, 1m),executor\n| order by TimeGenerated asc nulls last"
},
{
"displayName": "File System Bytes Write Per Executor",
"query": "SparkMetric_CL\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| where name_s contains \"executor.filesystem.file.write_bytes\" \n| summarize FileSystemWriteBytes=percentile(value_d,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last "
},
{
"displayName": "Task Scheduler Delay Latency",
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend schedulerdelay = Task_Info_Launch_Time_d - Stage_Info_Submission_Time_d\n| extend name=strcat(\"SchedulerDelayTime \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(schedulerdelay,90) , percentile(Task_Metrics_Executor_Run_Time_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
},
{
"displayName": "Streaming Errors Per Stream",
"query": "SparkListenerEvent_CL\r\n| extend slice = strcat(\"CountExceptions\",progress_name_s) \r\n| where Level contains \"Error\"\r\n| summarize count(Level) by bin(TimeGenerated, 1m), slice \r\n"
},
{
"displayName": "Shuffle Client Memory Per Executor",
"query": "SparkMetric_CL\r\n| where name_s contains \"shuffle-client.usedDirectMemory\"\r\n| extend sname=split(name_s, \".\")\r\n| extend executor=strcat(\"executorid:\",sname[1])\r\n| summarize percentile(value_d,90) by bin(TimeGenerated, 1m), executor\r\n| order by TimeGenerated asc nulls last"
},
{
"displayName": "Job Latency Per Job (Batch Duration)",
"query": "let results=SparkListenerEvent_CL\r\n| where Event_s contains \"SparkListenerJobStart\"\r\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Job_ID_d,apptag,Properties_spark_databricks_clusterUsageTags_clusterName_s,\r\nSubmission_Time_d,TimeGenerated\r\n| order by TimeGenerated asc nulls last \r\n| join kind= inner (\r\n SparkListenerEvent_CL\r\n | where Event_s contains \"SparkListenerJobEnd\"\r\n | where Job_Result_Result_s contains \"JobSucceeded\"\r\n | project Event_s,Job_ID_d,Completion_Time_d,TimeGenerated\r\n) on Job_ID_d;\r\nresults\r\n| extend slice=strcat(Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag)\r\n| extend jobDuration=Completion_Time_d - Submission_Time_d \r\n| summarize percentiles(jobDuration,10,30,50,90) by bin(TimeGenerated, 1m), slice\r\n| order by TimeGenerated asc nulls last"
},
{
"displayName": "Task Executor Compute Time (Data Skew Time)",
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last\n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"ExecutorComputeTime \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Executor_Run_Time_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
},
{
"displayName": "Streaming Throughput Input Rows/Sec",
"query": "SparkListenerEvent_CL\r\n| where Event_s contains \"progress\"\r\n| extend sname=strcat(progress_name_s,\"-inputRowsPerSecond\") \r\n| extend status = todouble(extractjson(\"$.[0].inputRowsPerSecond\", progress_sources_s))\r\n| summarize percentile(status,90) by bin(TimeGenerated, 1m) , sname\r\n| order by TimeGenerated asc nulls last \n"
},
{
"displayName": "Task Shuffle Bytes Read",
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last\n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"SchuffleBytesRead \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
},
{
"displayName": "Shuffle Memory Bytes Spilled Per Executor",
"query": "let results=SparkMetric_CL\n| where name_s contains \"executor.memoryBytesSpilled\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkMetric_CL\n | where name_s contains \"executor.memoryBytesSpilled\"\n | extend sname=split(name_s, \".\") \n | extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\n) on executor, TimeGenerated;\nresults\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \n| summarize any(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n"
},
{
"displayName": "% JVM Time Per Executor",
"query": "let results = SparkMetric_CL\n| where name_s contains \"executor.jvmGCTime\" \n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , jvmgcTime=count_d , executor ,name_s\n| join kind= inner (\nSparkMetric_CL\n| where name_s contains \"executor.RunTime\"\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , runTime=count_d , executor ,name_s\n) on executor, TimeGenerated;\nresults\n| extend JvmcpuUsage=(jvmgcTime/runTime)*100\n| summarize JvmCpuTime = percentile(JvmcpuUsage,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n| render timechart \n"
},
{
"displayName": "Running Executors",
"query": "SparkMetric_CL\n| where name_s !contains \"driver\" \n| where name_s contains \"executor\"\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[1]) \n| extend app=strcat(sname[0])\n| summarize NumExecutors=dcount(executor) by bin(TimeGenerated, 1m),app\n| order by TimeGenerated asc nulls last"
},
{
"displayName": "Shuffle Bytes Read To Disk Per Executor",
"query": "let results=SparkMetric_CL\r\n| where name_s contains \"executor.shuffleRemoteBytesReadToDisk\"\r\n| extend sname=split(name_s, \".\") \r\n| extend executor=strcat(\"executorid:\",sname[1])\r\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \r\n| order by TimeGenerated asc nulls last \r\n| join kind= inner (\r\n SparkMetric_CL\r\n | where name_s contains \"executor.shuffleRemoteBytesReadToDisk\"\r\n | extend sname=split(name_s, \".\") \r\n | extend executor=strcat(\"executorid:\",sname[1])\r\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\r\n) on executor, TimeGenerated;\r\nresults\r\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \r\n| summarize any(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\r\n| order by TimeGenerated asc nulls last\r\n"
},
{
"displayName": "Task Latency Per Stage (Tasks Duration)",
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,apptag,Properties_spark_databricks_clusterUsageTags_clusterName_s,Event_s,TimeGenerated\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Info_Finish_Time_d\n ) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend TaskLatency = Task_Info_Finish_Time_d - Task_Info_Launch_Time_d\n| extend slice=strcat(Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\"-\",Stage_Info_Stage_Name_s)\n| summarize percentile(TaskLatency,90) by bin(TimeGenerated,1m),slice\n| order by TimeGenerated asc nulls last;\n"
},
{
"displayName": "Task Throughput (Sum Of Tasks Per Stage)",
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Stage_ID_d,Task_Info_Task_ID_d,\n TaskEvent=Event_s,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend slice=strcat(\"#TasksCompleted \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize count(TaskEvent) by bin(TimeGenerated,1m),slice\n| order by TimeGenerated asc nulls last\n"
},
{
"displayName": "Shuffle Client Direct Memory",
"query": "SparkMetric_CL\n| where name_s contains \"shuffle-client.usedDirectMemory\"\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize percentile(value_d,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last"
},
{
"displayName": "Disk Bytes Spilled",
"query": "let results=SparkMetric_CL\n| where name_s contains \"executor.diskBytesSpilled\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkMetric_CL\n | where name_s contains \"executor.diskBytesSpilled\"\n | extend sname=split(name_s, \".\") \n | extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\n) on executor, TimeGenerated;\nresults\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \n| summarize any(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n"
},
{
"displayName": "Shuffle Bytes Read",
"query": "let results=SparkMetric_CL\n| where name_s contains \"executor.shuffleRemoteBytesReadToDisk\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkMetric_CL\n | where name_s contains \"executor.shuffleRemoteBytesReadToDisk\"\n | extend sname=split(name_s, \".\") \n | extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\n) on executor, TimeGenerated;\nresults\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \n| summarize any(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n"
}
]
},
"resources": [
{
"type": "Microsoft.OperationalInsights/workspaces",
"apiVersion": "2020-10-01",
"name": "[parameters('logAnalyticsWkspName')]",
"location": "[parameters('logAnalyticsWkspLocation')]",
"properties": {
"sku": {
"name": "[parameters('logAnalyticsWkspSku')]"
},
"retentionInDays": "[parameters('logAnalyticsWkspRentationDays')]",
"features": {
"enableDataExport": true
}
}
},
{
"copy": {
"name": "WkspSearch",
"count": "[length(variables('queries'))]"
},
"type": "Microsoft.OperationalInsights/workspaces/savedSearches",
"apiVersion": "2020-08-01",
"name": "[format('{0}/{1}', parameters('logAnalyticsWkspName'), guid(format('{0}{1}{2}', resourceGroup().id, deployment().name, copyIndex())))]",
"properties": {
"category": "Spark Metrics",
"displayName": "[variables('queries')[copyIndex()].displayName]",
"query": "[variables('queries')[copyIndex()].query]",
"version": 2,
"etag": "*"
},
"dependsOn": [
"[resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWkspName'))]"
]
}
],
"outputs": {
"logAnalyticsWkspId": {
"type": "string",
"value": "[reference(resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWkspName'))).customerId]"
},
"primarySharedKey": {
"type": "string",
"value": "[listKeys(resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWkspName')), '2020-10-01').primarySharedKey]"
},
"secondarySharedKey": {
"type": "string",
"value": "[listKeys(resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWkspName')), '2020-10-01').secondarySharedKey]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2019-10-01",
"name": "EventHub",
"resourceGroup": "[variables('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"namespaceName": {
"value": "[variables('eHNameSpace')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "17759276866562815474"
}
},
"parameters": {
"namespaceName": {
"type": "string",
"metadata": {
"description": "Name for the Event Hub cluster."
}
},
"eventHubName": {
"type": "string",
"defaultValue": "[parameters('namespaceName')]",
"metadata": {
"description": "Name for the Event Hub to be created in the Event Hub namespace within the Event Hub cluster."
}
},
"location": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Specifies the Azure location for all resources."
}
},
"eHRuleName": {
"type": "string",
"defaultValue": "rule",
"metadata": {
"description": ""
}
}
},
"functions": [],
"resources": [
{
"type": "Microsoft.EventHub/namespaces",
"apiVersion": "2021-01-01-preview",
"name": "[parameters('namespaceName')]",
"location": "[parameters('location')]",
"sku": {
"name": "Standard",
"tier": "Standard",
"capacity": 1
},
"properties": {
"isAutoInflateEnabled": false,
"maximumThroughputUnits": 0
}
},
{
"type": "Microsoft.EventHub/namespaces/eventhubs",
"apiVersion": "2021-01-01-preview",
"name": "[format('{0}/{1}', parameters('namespaceName'), parameters('eventHubName'))]",
"properties": {
"messageRetentionInDays": 7,
"partitionCount": 1
},
"dependsOn": [
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaceName'))]"
]
},
{
"type": "Microsoft.EventHub/namespaces/eventhubs/authorizationRules",
"apiVersion": "2021-01-01-preview",
"name": "[format('{0}/{1}/{2}', parameters('namespaceName'), parameters('eventHubName'), parameters('eHRuleName'))]",
"properties": {
"rights": [
"Send",
"Listen"
]
},
"dependsOn": [
"[resourceId('Microsoft.EventHub/namespaces/eventhubs', parameters('namespaceName'), parameters('eventHubName'))]"
]
}
],
"outputs": {
"eHNamespaceId": {
"type": "string",
"value": "[resourceId('Microsoft.EventHub/namespaces', parameters('namespaceName'))]"
},
"eHubNameId": {
"type": "string",
"value": "[resourceId('Microsoft.EventHub/namespaces/eventhubs', parameters('namespaceName'), parameters('eventHubName'))]"
},
"eHAuthRulesId": {
"type": "string",
"value": "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', split(format('{0}/{1}/{2}', parameters('namespaceName'), parameters('eventHubName'), parameters('eHRuleName')), '/')[0], split(format('{0}/{1}/{2}', parameters('namespaceName'), parameters('eventHubName'), parameters('eHRuleName')), '/')[1], split(format('{0}/{1}/{2}', parameters('namespaceName'), parameters('eventHubName'), parameters('eHRuleName')), '/')[2])]"
},
"eHPConnString": {
"type": "string",
"value": "[listKeys(resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', parameters('namespaceName'), parameters('eventHubName'), parameters('eHRuleName')), '2021-01-01-preview').primaryConnectionString]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2019-10-01",
"name": "PrivateEndPoints",
"resourceGroup": "[variables('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"keyvaultName": {
"value": "KeyVault"
},
"keyvaultPrivateLinkResource": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'KeyVault'), '2019-10-01').outputs.keyvault_id.value]"
},
"privateLinkSubnetId": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVNET'), '2019-10-01').outputs.privatelinksubnet_id.value]"
},
"storageAccountName": {
"value": "StorageAccount"
},
"storageAccountPrivateLinkResource": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount'), '2019-10-01').outputs.storageaccount_id.value]"
},
"eventHubName": {
"value": "[variables('eventHubName')]"
},
"eventHubPrivateLinkResource": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub'), '2019-10-01').outputs.eHNamespaceId.value]"
},
"targetSubResourceDfs": {
"value": "dfs"
},
"targetSubResourceVault": {
"value": "vault"
},
"targetSubResourceEventHub": {
"value": "namespace"
},
"vnetName": {
"value": "[parameters('spokeVnetName')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.4.412.5873",
"templateHash": "15615926081308867171"
}
},
"parameters": {
"storageAccountPrivateLinkResource": {
"type": "string",
"metadata": {
"description": "Storage Account Privatelink Resource"
}
},
"storageAccountName": {
"type": "string",
"metadata": {
"description": "Storage Account name"
}
},
"keyvaultPrivateLinkResource": {
"type": "string",
"metadata": {
"description": "Keyvault Private Link resource."
}
},
"keyvaultName": {
"type": "string",
"metadata": {
"description": "keyvault name."
}
},
"eventHubName": {
"type": "string",
"metadata": {
"description": "event hub name."
}
},
"eventHubPrivateLinkResource": {
"type": "string",
"metadata": {
"description": "EventHub Private Link resource."
}
},
"targetSubResourceDfs": {
"type": "string",
"metadata": {
"description": "Privatelink target sub resource DFS"
}
},
"targetSubResourceVault": {
"type": "string",
"metadata": {
"description": "Privatelink target sub resource vault"
}
},
"targetSubResourceEventHub": {
"type": "string",
"metadata": {
"description": "Privatelink target sub resource Event Hub"
}
},
"vnetName": {
"type": "string",
"metadata": {
"description": "Vnet name for private link"
}
},
"privateLinkSubnetId": {
"type": "string",
"metadata": {
"description": "Privatelink subnet Id"
}
},
"privateLinkLocation": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Privatelink subnet Id"
}
}
},
"functions": [],
"variables": {
"privateDnsNameStorage_var": "[format('privatelink.dfs.{0}', environment().suffixes.storage)]",
"storageAccountPrivateEndpointName_var": "[format('{0}privateendpoint', parameters('storageAccountName'))]",
"privateDnsNameVault_var": "privatelink.vaultcore.azure.net",
"keyvaultPrivateEndpointName_var": "[format('{0}privateendpoint', parameters('keyvaultName'))]",
"privateDnsNameEventHub_var": "privatelink.servicebus.windows.net",
"eventHubPrivateEndpointName_var": "[format('{0}privateendpoint', parameters('eventHubName'))]"
},
"resources": [
{
"type": "Microsoft.Network/privateEndpoints",
"apiVersion": "2021-02-01",
"name": "[variables('storageAccountPrivateEndpointName_var')]",
"location": "[parameters('privateLinkLocation')]",
"properties": {
"privateLinkServiceConnections": [
{
"name": "[variables('storageAccountPrivateEndpointName_var')]",
"properties": {
"privateLinkServiceId": "[parameters('storageAccountPrivateLinkResource')]",
"groupIds": [
"[parameters('targetSubResourceDfs')]"
]
}
}
],
"subnet": {
"id": "[parameters('privateLinkSubnetId')]"
}
},
"tags": {}
},
{
"type": "Microsoft.Network/privateDnsZones",
"apiVersion": "2020-06-01",
"name": "[variables('privateDnsNameStorage_var')]",
"location": "global",
"tags": {},
"properties": {},
"dependsOn": [
"[resourceId('Microsoft.Network/privateEndpoints', variables('storageAccountPrivateEndpointName_var'))]"
]
},
{
"type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks",
"apiVersion": "2020-06-01",
"name": "[format('{0}/{1}', variables('privateDnsNameStorage_var'), parameters('vnetName'))]",
"location": "global",
"properties": {
"virtualNetwork": {
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
},
"registrationEnabled": false
},
"dependsOn": [
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorage_var'))]"
]
},
{
"type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
"apiVersion": "2021-02-01",
"name": "[format('{0}/{1}', variables('storageAccountPrivateEndpointName_var'), 'default')]",
"properties": {
"privateDnsZoneConfigs": [
{
"name": "privatelink-dfs-core-windows-net",
"properties": {
"privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorage_var'))]"
}
}
]
},
"dependsOn": [
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorage_var'))]",
"[resourceId('Microsoft.Network/privateEndpoints', variables('storageAccountPrivateEndpointName_var'))]"
]
},
{
"type": "Microsoft.Network/privateEndpoints",
"apiVersion": "2021-02-01",
"name": "[variables('keyvaultPrivateEndpointName_var')]",
"location": "[parameters('privateLinkLocation')]",
"properties": {
"privateLinkServiceConnections": [
{
"name": "[variables('keyvaultPrivateEndpointName_var')]",
"properties": {
"privateLinkServiceId": "[parameters('keyvaultPrivateLinkResource')]",
"groupIds": [
"[parameters('targetSubResourceVault')]"
]
}
}
],
"subnet": {
"id": "[parameters('privateLinkSubnetId')]"
}
},
"tags": {}
},
{
"type": "Microsoft.Network/privateDnsZones",
"apiVersion": "2020-06-01",
"name": "[variables('privateDnsNameVault_var')]",
"location": "global",
"tags": {},
"properties": {},
"dependsOn": [
"[resourceId('Microsoft.Network/privateEndpoints', variables('keyvaultPrivateEndpointName_var'))]"
]
},
{
"type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks",
"apiVersion": "2020-06-01",
"name": "[format('{0}/{1}', variables('privateDnsNameVault_var'), parameters('vnetName'))]",
"location": "global",
"properties": {
"virtualNetwork": {
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
},
"registrationEnabled": false
},
"dependsOn": [
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameVault_var'))]"
]
},
{
"type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
"apiVersion": "2021-02-01",
"name": "[format('{0}/{1}', variables('keyvaultPrivateEndpointName_var'), 'default')]",
"properties": {
"privateDnsZoneConfigs": [
{
"name": "privatelink-vaultcore-azure-net",
"properties": {
"privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameVault_var'))]"
}
}
]
},
"dependsOn": [
"[resourceId('Microsoft.Network/privateEndpoints', variables('keyvaultPrivateEndpointName_var'))]",
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameVault_var'))]"
]
},
{
"type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks",
"apiVersion": "2020-06-01",
"name": "[format('{0}/{1}', variables('privateDnsNameEventHub_var'), parameters('vnetName'))]",
"location": "global",
"properties": {
"virtualNetwork": {
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
},
"registrationEnabled": false
},
"dependsOn": [
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameEventHub_var'))]"
]
},
{
"type": "Microsoft.Network/privateEndpoints",
"apiVersion": "2021-02-01",
"name": "[variables('eventHubPrivateEndpointName_var')]",
"location": "[parameters('privateLinkLocation')]",
"properties": {
"privateLinkServiceConnections": [
{
"name": "[variables('eventHubPrivateEndpointName_var')]",
"properties": {
"privateLinkServiceId": "[parameters('eventHubPrivateLinkResource')]",
"groupIds": [
"[parameters('targetSubResourceEventHub')]"
]
}
}
],
"subnet": {
"id": "[parameters('privateLinkSubnetId')]"
}
}
},
{
"type": "Microsoft.Network/privateDnsZones",
"apiVersion": "2020-06-01",
"name": "[variables('privateDnsNameEventHub_var')]",
"location": "global",
"tags": {},
"properties": {},
"dependsOn": [
"[resourceId('Microsoft.Network/privateEndpoints', variables('eventHubPrivateEndpointName_var'))]"
]
},
{
"type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
"apiVersion": "2021-02-01",
"name": "[format('{0}/{1}', variables('eventHubPrivateEndpointName_var'), 'default')]",
"properties": {
"privateDnsZoneConfigs": [
{
"name": "privatelink-servicebus-windows-net",
"properties": {
"privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameEventHub_var'))]"
}
}
]
},
"dependsOn": [
"[resourceId('Microsoft.Network/privateEndpoints', variables('eventHubPrivateEndpointName_var'))]",
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameEventHub_var'))]"
]
}
]
}
},
"dependsOn": [
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount')]",
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub')]",
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'KeyVault')]",
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]",
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVNET')]"
]
}
],
"outputs": {
"resourceGroupName": {
"type": "string",
"value": "[variables('resourceGroupName')]"
},
"keyVaultName": {
"type": "string",
"value": "[variables('keyVaultName')]"
},
"adbWorkspaceName": {
"type": "string",
"value": "[variables('adbWorkspaceName')]"
},
"storageAccountName": {
"type": "string",
"value": "[variables('storageAccountName')]"
},
"storageKey1": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount'), '2019-10-01').outputs.key1.value]"
},
"storageKey2": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount'), '2019-10-01').outputs.key2.value]"
},
"databricksWksp": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'DatabricksWorkspace'), '2019-10-01').outputs.databricks_workspace_id.value]"
},
"databricks_workspaceUrl": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'DatabricksWorkspace'), '2019-10-01').outputs.databricks_workspaceUrl.value]"
},
"keyvault_id": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'KeyVault'), '2019-10-01').outputs.keyvault_id.value]"
},
"keyvault_uri": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'KeyVault'), '2019-10-01').outputs.keyvault_uri.value]"
},
"logAnalyticsWkspId": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'loganalytics'), '2019-10-01').outputs.logAnalyticsWkspId.value]"
},
"logAnalyticsprimarySharedKey": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'loganalytics'), '2019-10-01').outputs.primarySharedKey.value]"
},
"logAnalyticssecondarySharedKey": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'loganalytics'), '2019-10-01').outputs.secondarySharedKey.value]"
},
"eHNamespaceId": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub'), '2019-10-01').outputs.eHNamespaceId.value]"
},
"eHubNameId": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub'), '2019-10-01').outputs.eHubNameId.value]"
},
"eHAuthRulesId": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub'), '2019-10-01').outputs.eHAuthRulesId.value]"
},
"eHPConnString": {
"type": "string",
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub'), '2019-10-01').outputs.eHPConnString.value]"
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment