Last active
October 15, 2019 15:45
-
-
Save DamolAAkinleye/0ecfa8205cc0428aaf3c6d15ac7a0f24 to your computer and use it in GitHub Desktop.
Created with Copy to Gist
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
param( | |
[string] | |
$ParamFile = "common.parameters.txt", | |
[string] | |
$subscriptionId, | |
[string] | |
$certPassword, | |
[string] | |
$sparkPassword, | |
[string] | |
$sparkSshPassword, | |
[string] | |
$sfPassword, | |
[string] | |
$resourceGroupName, | |
[ValidatePattern('^[a-zA-Z0-9_.-]*$')] | |
[ValidateLength(0, 40)] | |
[string] | |
$productName, | |
[string] | |
$resourceGroupLocation, | |
[ValidateSet("EastUS", "SouthCentralUS", "NorthEurope", "WestEurope", "SoutheastAsia", "WestUS2", "CanadaCentral", "CentralIndia")] | |
[string] | |
$resourceLocationForMicrosoftInsights, | |
[string] | |
$resourceLocationForServiceFabric, | |
[ValidateScript({Test-Path $_ })] | |
[string] | |
$deploymentCommonPath, | |
[ValidateSet("y", "n")] | |
[string] | |
$generateAndUseSelfSignedCerts, | |
[string] | |
$mainCert, | |
[string] | |
$reverseProxyCert, | |
[string] | |
$sslCert, | |
[ValidateSet("y", "n")] | |
[string] | |
$installModules, | |
[ValidateSet("y", "n")] | |
[string] | |
$resourceCreation, | |
[ValidateSet("y", "n")] | |
[string] | |
$sparkCreation, | |
[ValidateSet("y", "n")] | |
[string] | |
$serviceFabricCreation, | |
[ValidateSet("y", "n")] | |
[string] | |
$setupSecrets, | |
[ValidateSet("y", "n")] | |
[string] | |
$setupCosmosDB, | |
[ValidateSet("y", "n")] | |
[string] | |
$setupKVAccess | |
) | |
$user = [Security.Principal.WindowsIdentity]::GetCurrent(); | |
$ret = (New-Object Security.Principal.WindowsPrincipal $user).IsInRole([Security.Principal.WindowsBuiltinRole]::Administrator) | |
if (!$ret) { | |
Write-Host "The current command prompt session is not running as Administrator. Start command prompt by using the Run as Administrator option, and then try running the script again." | |
Exit 50 | |
} | |
$ErrorActionPreference = "stop" | |
Get-Content $ParamFile | Foreach-Object { | |
$l = $_.Trim() | |
if ($l.startsWith('#') -or $l.startsWith('//') -or !$l) { | |
return | |
} | |
$var = $l.Split('=', 2) | |
set-Variable -Name $var[0] -Value $var[1] | |
} | |
if ($deployResources -ne 'y') { | |
Write-Host "deployResources parameter value is not 'y'. This script will not execute." | |
Exit 0 | |
} | |
if ($generateNewSelfSignedCerts -eq 'n' -and !$certPassword) { | |
Write-Host "Please provide certPassword to import the existing certs" | |
Exit 40 | |
} | |
Remove-Item -path ".\cachedVariables" -Force -ErrorAction SilentlyContinue | |
$rootFolderPath = $PSScriptRoot | |
Import-Module "..\Deployment.Common\Helpers\UtilityModule" -ArgumentList $rootFolderPath, $resourceGroupName, $productName, $sparkClusterName, $randomizeProductName, $serviceFabricClusterName, $serviceAppName, $clientAppName, $sparkPassword, $sparkSshPassword, $sfPassword, $certPassword, $redisCacheSize -WarningAction SilentlyContinue | |
Set-Content -Path ".\cachedVariables" -NoNewline -Value $name | |
function Install-Modules { | |
Write-Host -ForegroundColor Green "Checking Module... " | |
Write-Host -ForegroundColor Green "Estimated time to complete: 5 mins" | |
Write-Host "Note: If any module is installed, you will have to close this prompt and restart deploy.bat as admin again... " | |
$modules = New-Object 'System.Collections.Generic.Dictionary[String,String]' | |
$modules.Add("azurerm", "6.13.1") | |
$modules.Add("azuread", "2.0.2.4") | |
$modules.Add("mdbc", " 5.1.4") | |
$moduleInstalled = $false | |
$modules.Keys | foreach { | |
if (!(Get-installedModule -name $_ -MinimumVersion $modules.Item($_) -ErrorAction SilentlyContinue )) { | |
Write-Host "Install Module: " $_ | |
$moduleInstalled = $true | |
Install-Module -Name $_ -Force -AllowClobber -Scope CurrentUser -Repository PSGallery | |
} | |
} | |
if ($moduleInstalled) { | |
Write-Host -ForegroundColor Yellow "The script execution completed after one or more packages have been installed. In order to use the latest packages, please close this prompt, open a new command prompt as admin and run deploy.bat again" | |
Exit 10 | |
} | |
} | |
# Check if file paths exist | |
function Check-FilePath { | |
$notFound= | |
$paths = @("$deploymentCommonPath\CosmosDB", "$deploymentCommonPath\Scripts", "$deploymentCommonPath\Resources") | |
foreach ($p in $paths) { | |
if (!(Test-Path $p)) { | |
Write-Host "$p does not exist" | |
Exit 20 | |
} | |
} | |
} | |
# Generate tokens with the actual values | |
function Get-Tokens { | |
$tokens = Get-DefaultTokens | |
# Template | |
$tokens.Add('subscriptionId', $subscriptionId ) | |
$tokens.Add('resourceLocationForMicrosoftInsights', $resourceLocationForMicrosoftInsights ) | |
$tokens.Add('tenantId', $tenantId ) | |
$tokens.Add('userId', $userId ) | |
$sparkType = 'hdinsight' | |
$keyvaultPrefix = 'keyvault' | |
$dataxJobTemplate = 'DataXDirect' | |
$dataxKafkaJobTemplate = 'kafkaDataXDirect' | |
$dataxBatchJobTemplate = 'DataXBatch' | |
if ($useDatabricks -eq 'y') { | |
$sparkType = 'databricks' | |
$keyvaultPrefix = 'secretscope' | |
$dataxJobTemplate = 'DataXDirectDatabricks' | |
$dataxKafkaJobTemplate = 'kafkaDataXDirectDatabricks' | |
$dataxBatchJobTemplate = 'DataXBatchDatabricks' | |
$tokens.Add('databricksClusterSparkVersion', $databricksClusterSparkVersion) | |
$tokens.Add('databricksClusterNodeType', $databricksClusterNodeType) | |
$tokens.Add('databricksSku', $databricksSku) | |
$tokens.Add('dbResourceGroupName', $resourceGroupName) | |
} | |
$tokens.Add('sparkType', $sparkType) | |
$tokens.Add('keyvaultPrefix', $keyvaultPrefix) | |
$tokens.Add('dataxJobTemplate', $dataxJobTemplate) | |
$tokens.Add('dataxKafkaJobTemplate', $dataxKafkaJobTemplate) | |
$tokens.Add('dataxBatchJobTemplate', $dataxBatchJobTemplate) | |
# CosmosDB | |
$tokens.Add('blobopsconnectionString', $blobopsconnectionString ) | |
$tokens.Add('configgenClientId', $azureADApplicationConfiggenApplicationId ) | |
$tokens.Add('configgenTenantId', $tenantName ) | |
# SF Template | |
$tokens.Add('certPrimaryThumbprint', $certPrimary.Certificate.Thumbprint ) | |
$tokens.Add('certPrimarySecretId', $certPrimary.SecretId ) | |
$tokens.Add('certReverseProxyThumbprint', $certReverseProxy.Certificate.Thumbprint ) | |
$tokens.Add('certReverseProxySecretId', $certReverseProxy.SecretId ) | |
$tokens.Add('sslcertThumbprint', $certSSL.Certificate.Thumbprint ) | |
$tokens.Add('resourceLocationForServiceFabric', $resourceLocationForServiceFabric ) | |
$tokens.Add('vmNodeTypeSize', $vmNodeTypeSize ) | |
$tokens.Add('vmNodeinstanceCount', $vmNodeinstanceCount ) | |
# Spark Template | |
$tokens.Add('vmSizeSparkHeadnode', $vmSizeSparkHeadnode ) | |
$tokens.Add('minInstanceCountSparkHeadnode', $minInstanceCountSparkHeadnode ) | |
$tokens.Add('targetInstanceCountSparkHeadnode', $targetInstanceCountSparkHeadnode ) | |
$tokens.Add('vmSizeSparkWorkernode', $vmSizeSparkWorkernode ) | |
$tokens.Add('targetInstanceCountSparkWorkernode', $targetInstanceCountSparkWorkernode ) | |
# Service param | |
$tokens.Add('writerRole', $writerRole ) | |
$tokens.Add('readerRole', $readerRole ) | |
$tokens.Add('serviceSecretPrefix', $serviceSecretPrefix ) | |
$tokens.Add('clientSecretPrefix', $clientSecretPrefix ) | |
$tokens.Add('serviceAppId', $azureADApplicationConfiggenApplicationId ) | |
$tokens.Add('clientAppId', $azureADApplicationApplicationId ) | |
$tokens.Add('azureADApplicationConfiggenResourceId', $azureADApplicationConfiggenResourceId ) | |
$aiKey = '' | |
$appInsight = Get-AzureRmApplicationInsights -resourceGroupName $resourceGroupName -Name $appInsightsName -ErrorAction SilentlyContinue | |
if ($appInsight) { | |
$aiKey = $appInsight.InstrumentationKey | |
} | |
$tokens.Add('appInsightKey', $aiKey ) | |
$tokens.Add('resourceLocation', $resourceGroupLocation ) | |
$certtype = '' | |
if ($useSelfSignedCerts -eq 'y') { | |
$certtype = 'test' | |
} | |
$tokens.Add('certtype', $certtype ) | |
$kafkaNativeConnectionString = '' | |
$kafkaNativeTopics = '' | |
if($enableKafkaSample -eq 'y') { | |
$kafkaNativeConnectionString = "datagen-kafkaNativeConnectionString" | |
$kafkaNativeTopics = "kafka1,kafka2" | |
} | |
$tokens.Add('kafkaNativeConnectionString', $kafkaNativeConnectionString) | |
$tokens.Add('kafkaNativeTopics', $kafkaNativeTopics) | |
$tokens | |
} | |
# Get appRole definition | |
function Create-AppRole([string] $Name, [string] $AppName, [string] $Description) { | |
$appRole = New-Object Microsoft.Open.AzureAD.Model.AppRole | |
$appRole.AllowedMemberTypes = New-Object System.Collections.Generic.List[string] | |
$appRole.AllowedMemberTypes.Add("User"); | |
if (($Name -eq $writerRole) -and ($AppName -eq $serviceAppName)) { | |
$appRole.AllowedMemberTypes.Add("Application"); | |
} | |
$appRole.DisplayName = $Name | |
$appRole.Id = New-Guid | |
$appRole.IsEnabled = $true | |
$appRole.Description = $Description | |
$appRole.Value = $Name | |
$appRole | |
} | |
# Add appRoles to AAD app | |
function Set-AzureAADAppRoles([string]$AppName) { | |
$role_r = Create-AppRole -Name $readerRole -AppName $AppName -Description $readerRole + " have ability to view flows" | |
$role_w = Create-AppRole -Name $writerRole -AppName $AppName -Description $writerRole + " can manage flows" | |
$roles = @($role_r, $role_W) | |
$app = Get-AzureADApplication -Filter "DisplayName eq '$AppName'" | |
if ($app.AppRoles) { | |
foreach($r in $roles) | |
{ | |
$role = $app.AppRoles | Where-Object { $_.Value -match $r.Value } | |
if (!$role) { | |
$app.AppRoles.Add($r) | |
Set-AzureADApplication -ObjectId $app.ObjectId -AppRoles $app.AppRoles | |
} | |
} | |
} | |
else { | |
foreach($r in $roles) | |
{ | |
$app.AppRoles.Add($r) | Out-Null | |
} | |
Set-AzureADApplication -ObjectId $app.ObjectId -AppRoles $app.AppRoles | |
} | |
} | |
# Add user with appRoles to service principal | |
function Add-UserAppRole([string]$AppName) { | |
$sp = Get-AzureADServicePrincipal -Filter "DisplayName eq '$AppName'" | |
$appRole = $sp.AppRoles | Where-Object { $_.Value -match $writerRole } | |
try { | |
New-AzureADUserAppRoleAssignment -ObjectId $userId -PrincipalId $userId -ResourceId $sp.ObjectId -Id $appRole.Id | |
} | |
catch {} | |
} | |
# Set secret to AAD app | |
function Set-AzureAADAppSecret([string]$AppName) { | |
$app = Get-AzureRmADApplication -DisplayName $AppName | |
if ($app) | |
{ | |
$startDate = Get-Date | |
$endDate = $startDate.AddYears(2) | |
$keyValue = New-AzureADApplicationPasswordCredential -ObjectId $app.ObjectId -StartDate $startDate -EndDate $endDate | |
} | |
$keyValue | |
} | |
# Set credential to AAD app | |
function Set-AzureAADAppCert([string]$AppName) { | |
$app = Get-AzureRmADApplication -DisplayName $AppName | |
if ($app) | |
{ | |
$cer = $certPrimary.Certificate | |
$certValue = [System.Convert]::ToBase64String($cer.GetRawCertData()) | |
az ad app credential reset --append --id $app.ApplicationId --cert $certValue | |
} | |
} | |
# Set secret to AAD app | |
function Generate-AADApplication([string]$appName, [string]$websiteName) { | |
$app = Get-AzureRmADApplication -DisplayName $appName | |
if (!$app) | |
{ | |
if ($websiteName){ | |
$app = New-AzureRmADApplication -DisplayName $appName -IdentifierUris "https://$tenantName/$appName" -ReplyUrls "https://$websiteName.azurewebsites.net/authReturn" | |
} | |
else { | |
$app = New-AzureRmADApplication -DisplayName $appName -IdentifierUris "https://$tenantName/$appName" | |
} | |
} | |
if ($app) | |
{ | |
$urls = $app.IdentifierUris | |
if ($urls.Count -eq 0) { | |
Set-AzureRmADApplication -ObjectId $app.ObjectId -IdentifierUris "https://$tenantName/$appName" -ErrorAction SilentlyContinue | |
} | |
} | |
if ($websiteName) | |
{ | |
$urls = $app.ReplyUrls | |
$urls.Add("https://$websiteName.azurewebsites.net/authReturn") | |
Set-AzureRmADApplication -ObjectId $app.ObjectId -ReplyUrl $urls -ErrorAction SilentlyContinue | |
} | |
$servicePrincipal = Get-AzureRmADServicePrincipal -ApplicationId $app.ApplicationId | |
if (!$servicePrincipal) | |
{ | |
$servicePrincipal = New-AzureRmADServicePrincipal -ApplicationId $app.ApplicationId | |
} | |
$app | |
} | |
# Generate SelfSigned Certs | |
function Generate-SelfSignedCert([string] $certFileName, [string] $outputPath) { | |
$todaydt = Get-Date | |
$2years = $todaydt.AddYears(2) | |
$clustername = "$serviceFabricName" | |
$subject = "CN=$clustername"+ ".$resourceLocationForServiceFabric" + ".cloudapp.azure.com" | |
$certFilePath = Join-Path $outputPath $certFileName | |
$password = ConvertTo-SecureString $certPwd -AsPlainText -Force | |
$cert = New-SelfSignedCertificate -Subject $subject -notafter $2years -CertStoreLocation cert:\LocalMachine\My | |
# Export the cert to a PFX with password | |
Export-PfxCertificate -Cert "cert:\LocalMachine\My\$($cert.Thumbprint)" -FilePath $certFilePath -Password $password | Out-Null | |
Import-PfxCertificate -FilePath $certFilePath -CertStoreLocation cert:\CurrentUser\My -Password $password | Out-Null | |
$certFilePath | |
} | |
# Import Certs to keyVault | |
function Import-CertsToKeyVault([string]$certPath) { | |
$certPath = $certPath.Replace("""", "") | |
$certBaseName = ((Get-Item $certPath).Name).Replace(".", "") | |
$password = ConvertTo-SecureString $certPwd -AsPlainText -Force | |
# Upload to Key Vault | |
if ($serviceFabricCreation -eq 'y') { | |
$cert = Import-AzureKeyVaultCertificate -VaultName $sfKVName -Name $certBaseName -FilePath $certPath -Password $password | |
} | |
else { | |
$cert = Get-AzureKeyVaultCertificate -VaultName $sfKVName -Name $certBaseName | |
} | |
$cert | |
} | |
# Add script actions to Spark | |
function Add-ScriptActions { | |
$clusterName = "$sparkName" | |
$scriptActionName = "StartMSIServer" | |
$scAction = Get-AzureRmHDInsightScriptActionHistory -ClusterName $clusterName | |
if (($scAction.Name -eq "$scriptActionName") -and ($scAction.Status -eq 'succeeded')) { | |
return | |
} | |
$tokens = Get-Tokens | |
Deploy-Files -saName $sparkBlobAccountName -containerName "scripts" -filter "*.*" -filesPath "$deploymentCommonPath\Scripts" -targetPath "" -translate $True -tokens $tokens | |
$scriptActionUri = "https://$sparkBlobAccountName.blob.core.windows.net/scripts/startmsiserverservice.sh" | |
$nodeTypes = "headnode", "workernode" | |
Submit-AzureRmHDInsightScriptAction -ClusterName $clusterName ` | |
-Name $scriptActionName ` | |
-Uri $scriptActionUri ` | |
-NodeTypes $nodeTypes ` | |
-PersistOnSuccess | |
} | |
# Setup cosmosDB | |
function Setup-CosmosDB { | |
Connect-Mdbc -ConnectionString $dbCon -DatabaseName "production" | |
$colnames = @( | |
"commons", | |
"configgenConfigs", | |
"flows", | |
"sparkClusters" | |
"sparkJobs" | |
) | |
$colnames | foreach { | |
try{ | |
$response = Add-MdbcCollection -Name $_ | |
if (!$response) { | |
throw | |
} | |
} | |
catch | |
{} | |
} | |
$templatePath = "$deploymentCommonPath\CosmosDB" | |
$outputPath = Get-OutputFilePath | |
$qry = New-MdbcQuery -Name "_id" -Exists | |
$colnames | foreach{ | |
$colName = $_ | |
$collection1 = $Database.GetCollection($colName) | |
Remove-MdbcData -Query $qry -Collection $collection1 | |
$templateName = "$colName.json" | |
$outputFile = Join-Path $outputPath "_$templateName" | |
$filePath = Join-Path $templatePath $templateName | |
$tokens = Get-Tokens | |
$rawData = Get-Content -Raw -Path $filePath | |
$rawData = Translate-Tokens -Source $rawData -Tokens $tokens | |
$json = ConvertFrom-Json -InputObject $rawData | |
$json | foreach { | |
try | |
{ | |
$_ | ConvertTo-Json -Depth 10 | Set-Content -Encoding Unicode $outputFile | |
$input = Import-MdbcData $outputFile -FileFormat Json | |
$response = Add-MdbcData -InputObject $input -Collection $collection1 -NewId | |
if (!$response) { | |
throw | |
} | |
} | |
catch{} | |
} | |
} | |
Remove-Module Mdbc | |
} | |
# Create secrets to keyVaults for Spark | |
function Setup-SecretsForCert { | |
$vaultName = "$sfKVName" | |
$secretName = "certpassword" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $certPwd | |
} | |
# Create secrets to keyVaults for Spark | |
function Setup-SecretsForSpark { | |
$vaultName = "$sparkRDPKVName" | |
$secretName = "sparkLogin" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $sparkLogin | |
$secretName = "sparkclusterloginpassword" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $sparkPwd | |
$secretName = "sparksshuser" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $sparksshuser | |
$secretName = "sparksshpassword" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $sparkSshPwd | |
$vaultName = "$servicesKVName" | |
$prefix = "$serviceSecretPrefix-" | |
$secretName = $prefix + "livyconnectionstring-" + $sparkName | |
$tValue = "endpoint=https://$sparkName.azurehdinsight.net/livy;username=$sparkLogin;password=$sparkPwd" | |
if ($useDatabricks -eq 'y') { | |
$tValue = "endpoint=https://$resourceGroupLocation.azuredatabricks.net/api/2.0/;dbtoken=" | |
} | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tValue | |
} | |
# Create secrets to keyVaults for SF | |
function Setup-SecretsForServiceFabric { | |
$vaultName = "$fabricRDPKVName" | |
$secretName = "sfadminpassword" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $sfPwd | |
$secretName = "sfadminuser" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $sfadminuser | |
} | |
# Create secrets to keyVaults | |
function Setup-Secrets { | |
$vaultName = "$servicesKVName" | |
$prefix = "$serviceSecretPrefix-" | |
$secretName = $prefix + "configgenconfigs" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $dbCon | |
$secretName = $prefix + "configgenconfigsdatabasename" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value "production" | |
$secretName = $prefix + $configBlobAccountName + "-blobconnectionstring" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $blobopsconnectionString | |
$secretName = $prefix + $sparkBlobAccountName + "-blobconnectionstring" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $blobsparkconnectionString | |
$secretName = $prefix + "aiInstrumentationKey" | |
$aiKey = (Get-AzureRmApplicationInsights -resourceGroupName $resourceGroupName -Name $appInsightsName).InstrumentationKey | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $aiKey | |
$secretName = $prefix + "tenantid" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tenantName | |
$secretName = $prefix + "clientId" | |
$tValue = ($azureADApplicationConfiggenApplicationId.Split(" "))[0] | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tValue | |
$secretName = $prefix + "clientsecret" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $azureADAppSecretConfiggenValue | |
$secretName = $prefix + "eventhubnamespaceconnectionstring" | |
$tValue = (Invoke-AzureRmResourceAction -ResourceGroupName $resourceGroupName -ResourceType Microsoft.EventHub/namespaces/AuthorizationRules -ResourceName "$eventHubNamespaceName/listen" -Action listKeys -ApiVersion 2015-08-01 -Force).primaryConnectionString | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tValue | |
$secretName = $prefix + "azureservicesauthconnectionstring" | |
$tValue = "<Parameter Name=""AzureServicesAuthConnectionString"" Value=""RunAs=App;AppId=" + $azureADApplicationConfiggenApplicationId + ";TenantId=" + $tenantId + ";CertificateThumbprint=" + $certPrimary.Certificate.Thumbprint + ";CertificateStoreLocation=LocalMachine""/>" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tValue | |
$secretName = $prefix + "cacertificatelocation" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $caCertificateLocation | |
$prefix = "$clientSecretPrefix-" | |
$secretName = $prefix + "aiKey" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $aiKey | |
$secretName = $prefix + "clientId" | |
$tValue = ($azureADApplicationApplicationId.Split(" "))[0] | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tValue | |
$secretName = $prefix + "clientSecret" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $azureADAppSecretValue | |
$secretName = $prefix + "serviceClusterUrl" | |
$sfName = "$serviceFabricName" | |
$tValue = "https://$sfName"+ ".$resourceLocationForServiceFabric" + ".cloudapp.azure.com" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tValue | |
$secretName = $prefix + "serviceResourceId" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $azureADApplicationConfiggenResourceId | |
$secretName = $prefix + "mongoDbUrl" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value "test" | |
$secretName = $prefix + "redisDataConnectionString" | |
$redisKey = (Get-AzureRmRedisCacheKey -Name $redisName -resourceGroupName $resourceGroupName).PrimaryKey | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value "$redisName.redis.cache.windows.net:6380,password=$redisKey,ssl=True,abortConnect=False" | |
$secretName = $prefix + "sessionSecret" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value "test" | |
$secretName = $prefix + "subscriptionId" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $subscriptionId | |
$secretName = $prefix + "tenantName" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tenantName | |
$prefix = "" | |
$storageAccount = Get-AzureRmStorageAccount -resourceGroupName $resourceGroupName -Name $sparkBlobAccountName | |
$tValue = "" | |
if ($storageAccount.Context.ConnectionString -match 'AccountKey=(.*)') | |
{ | |
$tValue = $Matches[1].Replace("AccountKey=", "") | |
} | |
$secretName = $prefix + "datagen-storageaccountAccesskey" | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tValue | |
$vaultName = "$sparkKVName" | |
$secretName = $prefix + "datax-sa-" + $configBlobAccountName | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tValue | |
$storageAccount = Get-AzureRmStorageAccount -resourceGroupName $resourceGroupName -Name $sparkBlobAccountName | |
$tValue = "" | |
if ($storageAccount.Context.ConnectionString -match 'AccountKey=(.*)') | |
{ | |
$tValue = $Matches[1].Replace("AccountKey=", "") | |
} | |
$secretName = $prefix + "datax-sa-" + $sparkBlobAccountName | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tValue | |
$secretName = $prefix + "metric-eventhubconnectionstring" | |
$tValue = (Get-AzureRmEventHubKey -resourceGroupName $resourceGroupName -NamespaceName "$eventHubNamespaceName" -EventHubName metricseventhub -AuthorizationRuleName send).PrimaryConnectionString | |
Setup-Secret -VaultName $vaultName -SecretName $secretName -Value $tValue | |
} | |
# Setup keyVault access | |
function Setup-KVAccess { | |
# Get ObjectId of web app | |
$servicePrincipalId = az resource show -g $resourceGroupName --name $websiteName --resource-type Microsoft.Web/sites --query identity.principalId | |
# Get ObjectId of vmss | |
$vmssId = az resource show -g $resourceGroupName --name $vmNodeTypeName --resource-type Microsoft.Compute/virtualMachineScaleSets --query identity.principalId | |
# Get ObjectId of Service app | |
try { | |
$servicePrincipalConfiggenId = az ad sp list --display-name $serviceAppName --query [0].objectId | |
} | |
catch { | |
Write-Host "Error on getting the service principal objectId" | |
Exit 30 | |
} | |
az keyvault set-policy --name $servicesKVName --object-id $servicePrincipalId --secret-permissions get, list, set > $null 2>&1 | |
az keyvault set-policy --name $servicesKVName --object-id $servicePrincipalConfiggenId --secret-permissions get, list, set > $null 2>&1 | |
az keyvault set-policy --name $servicesKVName --object-id $vmssId --secret-permissions get, list, set > $null 2>&1 | |
az keyvault set-policy --name $sparkKVName --object-id $servicePrincipalId --secret-permissions get, list, set > $null 2>&1 | |
az keyvault set-policy --name $sparkKVName --object-id $servicePrincipalConfiggenId --secret-permissions get, list, set, delete > $null 2>&1 | |
az keyvault set-policy --name $sparkKVName --object-id $vmssId --secret-permissions get, list, set > $null 2>&1 | |
if($useDatabricks -eq 'n') { | |
# Get ObjectId of sparkManagedIdentityName | |
$SparkManagedIdentityId = az resource show -g $resourceGroupName --name $sparkManagedIdentityName --resource-type Microsoft.ManagedIdentity/userAssignedIdentities --query properties.principalId | |
az keyvault set-policy --name $servicesKVName --object-id $SparkManagedIdentityId --secret-permissions get, list, set > $null 2>&1 | |
az keyvault set-policy --name $sparkKVName --object-id $SparkManagedIdentityId --secret-permissions get, list, set > $null 2>&1 | |
} | |
} | |
# Import SSL Cert To Service Fabric | |
function Import-SSLCertToSF([string]$certPath) { | |
$certPath = $certPath.Replace("""", "") | |
$certBaseName = (Get-Item $certPath).BaseName | |
$clustername = "$serviceFabricName" | |
$bytes = [System.IO.File]::ReadAllBytes($certPath) | |
$base64 = [System.Convert]::ToBase64String($bytes) | |
$jsonBlob = @{ | |
data = $base64 | |
dataType = 'pfx' | |
password = $certPwd | |
} | ConvertTo-Json | |
$contentbytes = [System.Text.Encoding]::UTF8.GetBytes($jsonBlob) | |
$content = [System.Convert]::ToBase64String($contentbytes) | |
$secretValue = ConvertTo-SecureString -String $content -AsPlainText -Force | |
# Upload the certificate to the key vault as a secret | |
Write-Host "Writing secret to $certBaseName in vault $sfKVName" | |
$secret = Set-AzureKeyVaultSecret -VaultName $sfKVName -Name $certBaseName -SecretValue $secretValue | |
# Add a certificate to all the VMs in the cluster. | |
Add-AzureRmServiceFabricApplicationCertificate -resourceGroupName $resourceGroupName -Name $clustername -SecretIdentifier $secret.Id | Out-Null | |
} | |
# Open 443 port | |
function Open-Port { | |
$probename = "AppPortProbe6" | |
$rulename = "AppPortLBRule6" | |
$port = 443 | |
# Get the load balancer resource | |
$resource = Get-AzureRmResource | Where {$_.resourceGroupName -eq $resourceGroupName -and $_.ResourceType -eq "Microsoft.Network/loadBalancers" -and $_.Name -like "LB-*"} | |
$slb = Get-AzureRmLoadBalancer -Name $resource.Name -resourceGroupName $resourceGroupName | |
$probe = Get-AzureRmLoadBalancerProbeConfig -Name $probename -LoadBalancer $slb -ErrorAction SilentlyContinue | |
if (!($probe)) { | |
# Add a new probe configuration to the load balancer | |
$slb | Add-AzureRmLoadBalancerProbeConfig -Name $probename -Protocol Tcp -Port $port -IntervalInSeconds 15 -ProbeCount 2 | Out-Null | |
$probe = Get-AzureRmLoadBalancerProbeConfig -Name $probename -LoadBalancer $slb | |
} | |
$rule = get-AzureRmLoadBalancerRuleConfig -LoadBalancer $slb -Name $rulename -ErrorAction SilentlyContinue | |
if (!($rule)) { | |
# Add rule configuration to the load balancer | |
$slb | Add-AzureRmLoadBalancerRuleConfig -Name $rulename -BackendAddressPool $slb.BackendAddressPools[0] -FrontendIpConfiguration $slb.FrontendIpConfigurations[0] -Probe $probe -Protocol Tcp -FrontendPort $port -BackendPort $port | Out-Null | |
} | |
# Set the goal state for the load balancer | |
$slb | Set-AzureRmLoadBalancer | Out-Null | |
} | |
# Setup Service Fabric cluster. | |
# Import SSL cert and open 443 port | |
function Setup-SF { | |
Import-SSLCertToSF -certPath "$sslCert" | |
Open-Port | |
} | |
# Generate paramter files for the service deployment | |
function Translate-ParameterFiles([string] $ParameterFilePath, [system.collections.generic.dictionary[string,string]]$Tokens) { | |
$rawData = Get-Content -Raw -Path $ParameterFilePath | |
$rawData = Translate-Tokens -Source $rawData -Tokens $Tokens | |
$rawData | |
} | |
# Generate paramter files for the service deployment | |
function Generate-ParameterFiles([string] $parametersFolder = '', [string] $parametersOutputFolder = '') { | |
New-Item -ItemType Directory -Force -Path $parametersOutputFolder -ErrorAction SilentlyContinue | |
$allParameters = Get-ChildItem -Path $parametersFolder -Filter *.json -File -Recurse | |
$tokens = Get-Tokens | |
foreach($parameterFile in $allParameters) { | |
$translatedData = Translate-ParameterFiles -ParameterFilePath $parameterFile.FullName -Token $tokens | |
$filePath = [System.IO.Path]::Combine($parametersOutputFolder, $parameterFile.Name) | |
Set-Content -Path $filePath -Value $translatedData | |
} | |
} | |
# Prepare steps for the service deployment | |
function Prepare-AppDeployment { | |
$parametersFolder = Join-Path -Path $deploymentAppPath -ChildPath "Services\Parameters" | |
$parametersOutputFolder = Join-Path -Path $deploymentAppPath -ChildPath "Outputs\Services\Parameters" | |
Generate-ParameterFiles -parametersFolder $parametersFolder -parametersOutputFolder $parametersOutputFolder | |
} | |
# Prepare steps for the service deployment | |
function Prepare-AdminSteps { | |
$parameterFile = Join-Path $adminParameterPath "adminsteps.parameters.txt" | |
$parameterFileOutput = Join-Path $deploymentAppPath "adminsteps.parameters.txt" | |
$tokens = Get-Tokens | |
$translatedData = Translate-ParameterFiles -ParameterFilePath $parameterFile -Token $tokens | |
Set-Content -Path $parameterFileOutput -Value $translatedData | |
} | |
#****************************************************************************** | |
# Script body | |
# Execution begins here | |
#****************************************************************************** | |
$ErrorActionPreference = "stop" | |
Push-Location $PSScriptRoot | |
Write-Host -ForegroundColor Green "Total estimated time to complete: 2 to 4 hours" | |
if ($installModules -eq 'y') { | |
Install-Modules | |
} | |
Check-FilePath | |
# sign in | |
Write-Host "Logging in..." | |
# select subscription | |
Write-Host "Selecting subscription '$subscriptionId'" | |
Check-Credential -SubscriptionId $subscriptionId -TenantId $tenantId | |
Write-Host "Logging in for AzureAD" | |
$acc = Connect-AzureAD -TenantId $tenantId | |
$tenantName = $acc.Tenant.Domain | |
$tenantId = $acc.Tenant.Id.Guid | |
$userId = (az ad signed-in-user show --query 'objectId').Replace("""", "") | |
# Check login info | |
if (!($tenantName) -or !($tenantId) -or !($userId)) { | |
Write-Host "Error on getting tenantName, tenantId or userId" | |
Exit 100 | |
} | |
# Output login info | |
Write-Host "tenantId: " $tenantId | |
Write-Host "tenantName: " $tenantName | |
Write-Host "userId: " $userId | |
# Create or check for existing resource group | |
$resourceGroup = Get-AzureRmResourceGroup -Name $resourceGroupName -ErrorAction SilentlyContinue | |
if(!$resourceGroup) { | |
Write-Host "Resource group '$resourceGroupName' does not exist" | |
if(!$resourceGroupLocation) { | |
$resourceGroupLocation = Read-Host "resourceGroupLocation" | |
} | |
Write-Host "Creating resource group '$resourceGroupName' in location '$resourceGroupLocation'" | |
New-AzureRmResourceGroup -Name $resourceGroupName -Location $resourceGroupLocation | |
} | |
else { | |
Write-Host "Using existing resource group '$resourceGroupName'" | |
$resourceGroupLocation = $resourceGroup.Location | |
} | |
$templatePath = $resourcesTemplatePath | |
Write-Host -ForegroundColor Green "Starting deployment..." | |
if($resourceCreation -eq 'y') { | |
Write-Host -ForegroundColor Green "Deploying resources (1/16 steps): All resources except HDInsight and Service Fabric clusters will be deployed" | |
Write-Host -ForegroundColor Green "Estimated time to complete: 40 mins" | |
$tokens = Get-Tokens | |
Deploy-Resources -templateName "Resource-Template.json" -paramName "Resource-parameter.json" -templatePath $templatePath -tokens $tokens | |
} | |
if($sparkCreation -eq 'y') { | |
Write-Host -ForegroundColor Green "Deploying resources (2/16 steps): A spark cluster will be deployed" | |
Setup-SecretsForSpark | |
$tokens = Get-Tokens | |
if ($useDatabricks -eq 'n') { | |
Write-Host -ForegroundColor Green "Estimated time to complete: 20 mins" | |
Deploy-Resources -templateName "Spark-Template.json" -paramName "Spark-parameter.json" -templatePath $templatePath -tokens $tokens | |
} | |
else { | |
Write-Host -ForegroundColor Green "Estimated time to complete: 5 mins" | |
Deploy-Resources -templateName "Databricks-Template.json" -paramName "Databricks-Parameter.json" -templatePath $templatePath -tokens $tokens | |
} | |
} | |
# Preparing certs... | |
if ($generateNewSelfSignedCerts -eq 'y') { | |
Write-Host "Generating SelfSigned certs..." | |
New-Item -ItemType Directory -Force -Path $certPath -ErrorAction SilentlyContinue | |
$mainCert = Generate-SelfSignedCert -certFileName "certprimary$name.pfx" -outputPath $certPath | |
$reverseProxyCert = Generate-SelfSignedCert -certFileName "certreverseproxy$name.pfx" -outputPath $certPath | |
$sslCert = Generate-SelfSignedCert -certFileName "certssl$name.pfx" -outputPath $certPath | |
} | |
Write-Host "processing certs..." | |
$certPrimary = Import-CertsToKeyVault -certPath $mainCert | |
$certReverseProxy = Import-CertsToKeyVault -certPath $reverseProxyCert | |
$certSSL = Import-CertsToKeyVault -certPath $sslCert | |
#aad | |
Write-Host -ForegroundColor Green "processing AAD... (3/16 steps)" | |
Write-Host -ForegroundColor Green "Estimated time to complete: 2 mins" | |
$azureADApplication = Generate-AADApplication -appName $clientAppName -websiteName $websiteName | |
$azureADApplicationConfiggen = Generate-AADApplication -appName $serviceAppName | |
$azureADApplicationApplicationId = $azureADApplication.ApplicationId.Guid | |
$azureADApplicationConfiggenApplicationId = $azureADApplicationConfiggen.ApplicationId.Guid | |
$azureADApplicationConfiggenResourceId = $azureADApplicationConfiggen.IdentifierUris[0] | |
$azureADAppSecret = Set-AzureAADAppSecret -AppName $clientAppName | |
$azureADAppSecretConfiggen = Set-AzureAADAppSecret -AppName $serviceAppName | |
Set-AzureAADAppCert -AppName $serviceAppName | |
$azureADAppSecretValue = $azureADAppSecret.Value | |
$azureADAppSecretConfiggenValue = $azureADAppSecretConfiggen.Value | |
Set-AzureAADAppRoles -AppName $clientAppName | |
Set-AzureAADAppRoles -AppName $serviceAppName | |
Add-UserAppRole -AppName $clientAppName | |
Add-UserAppRole -AppName $serviceAppName | |
Set-AzureAADAccessControl -AppId $azureADApplicationConfiggenApplicationId | |
Set-AzureAADApiPermission -ServiceAppId $azureADApplicationConfiggenApplicationId -ClientAppId $azureADApplicationApplicationId -RoleName $writerRole | |
if($serviceFabricCreation -eq 'y') { | |
Write-Host -ForegroundColor Green "Deploying resources (4/16 steps): A Service fabric cluster will be deployed" | |
Write-Host -ForegroundColor Green "Estimated time to complete: 20 mins" | |
Setup-SecretsForServiceFabric | |
Setup-SecretsForCert | |
$tokens = Get-Tokens | |
Deploy-Resources -templateName "SF-Template.json" -paramName "SF-parameter.json" -templatePath $templatePath -tokens $tokens | |
} | |
# Processing | |
$dbCon = Get-CosmosDBConnectionString -Name $docDBName | |
$blobopsconnectionString = Get-StorageAccountConnectionString -Name $configBlobAccountName | |
$blobsparkconnectionString = Get-StorageAccountConnectionString -Name $sparkBlobAccountName | |
Write-Host "Prepare for the service deployment..." | |
Prepare-AppDeployment | |
Write-Host "Prepare for the admin steps..." | |
Prepare-AdminSteps | |
# Secrets | |
if ($setupSecrets -eq 'y') { | |
Write-Host -ForegroundColor Green "Setting up Secrets... (5/16 steps)" | |
Write-Host -ForegroundColor Green "Estimated time to complete: 1 min" | |
Setup-Secrets | |
} | |
# Spark | |
if ($sparkCreation -eq 'y') { | |
Write-Host -ForegroundColor Green "Setting up ScriptActions... (6/16 steps)" | |
if ($useDatabricks -eq 'n') { | |
Write-Host -ForegroundColor Green "Estimated time to complete: 2 mins" | |
Add-ScriptActions | |
} | |
} | |
# cosmosDB | |
if ($setupCosmosDB -eq 'y') { | |
Write-Host -ForegroundColor Green "Setting up CosmosDB... (7/16 steps)" | |
Write-Host -ForegroundColor Green "Estimated time to complete: 1 min" | |
Setup-CosmosDB | |
} | |
# Access Policies | |
if ($setupKVAccess -eq 'y') { | |
Write-Host -ForegroundColor Green "Setting up KV access... (8/16 steps)" | |
Write-Host -ForegroundColor Green "Estimated time to complete: 2 mins" | |
Setup-KVAccess | |
} | |
# setup SF | |
if ($serviceFabricCreation -eq 'y') { | |
Write-Host -ForegroundColor Green "Setting up SF... (9/16 steps)" | |
Write-Host -ForegroundColor Green "Estimated time to complete: 60 mins" | |
Setup-SF | |
} | |
# Clean up \Temp folder | |
Write-Host "Cleaning up the cache files..." | |
CleanUp-Folder -FolderName $tempPath | |
Exit 0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment