Deploy ADFv2 exported ARM templates

DeployADFv2ArmTemplate.ps1

# Based on 
# https://docs.microsoft.com/en-us/azure/data-factory/continuous-integration-deployment
# https://azure.microsoft.com/mediahandler/files/resourcefiles/whitepaper-adf-on-azuredevops/Azure%20data%20Factory-Whitepaper-DevOps.pdf
#
# WARNING: Deleting the pipelines will also delete the execution log. If you do need the log, consider exporting it to Azure Monitor
# as described https://docs.microsoft.com/en-us/azure/data-factory/monitor-using-azure-monitor. Also keep in mind that
# ADF stores pipeline-run data for only 45 days, so exporting to azure monitor will assist preserving the data for more time.
#
# Prerequisites
# This code runs on powershell 7+ to use ForEach-Object -Parallel.
# Place this script next to the linkedTemplates folder that you exported from ADF v2 UI or got from the adf_publish branch.
#
# Install-Module -Name Az -AllowClobber -Scope CurrentUser
# Set-ExecutionPolicy -Scope CurrentUser -ExecutionPolicy Bypass
#
# Login to the subscription
# $subscriptionId="the-guid-from-subscription"
# Connect-AzAccount -Subscription $subscriptionId

$storageResourceGroup = "the-resource-group-name-for-the-storage-to-upload-linked-arm"
$storageAccountName = "the-storage-account-name"
$storageContainerName = "the-container-within-the-storage-to-upload-linked-arm"

$deployResourceGroup="the-resource-group-that-contains-ADF-v2"
$deployDataFactoryName="the-ADF-v2-name"

# 2 hours expiration time for the sas token
$sasTokenTTL=2.0

# Location of the partial arm templates
$linkedTemplateFolder="$PSScriptRoot/linkedTemplates/"

$linkedTemplateMasterFile="$($linkedTemplateFolder)ArmTemplate_master.json"
$linkedTemplateParameterFile="$($linkedTemplateFolder)ArmTemplateParameters_master.json"

###### End of parameters ######

## Clean up existing ADF resources
## This is done to avoid having residuals from deleted resources since the deployment is going to be incremental

# Remove triggers
# If you don't want to remove them, you can comment out the the Remove.
# You will need to stop them, otherwise you will get the following error:
# Resource Microsoft.DataFactory/factories/triggers 'data-factory-name/Blob Created Trigger' failed with
# message '{   "error": {     "code": "TriggerEnabledCannotUpdate",     "message": "Cannot update enabled Trigger;
# the trigger needs to be disabled first. ",     "target": null,     "details": null   } }'
$triggersADF = Get-AzDataFactoryV2Trigger -DataFactoryName $deployDataFactoryName -ResourceGroupName $deployResourceGroup
if($null -ne $triggersADF)
{ 
  $triggersADF | ForEach-Object -Parallel { 
      Stop-AzDataFactoryV2Trigger -ResourceGroupName $using:deployResourceGroup -DataFactoryName $using:deployDataFactoryName -Name $_.Name -Force 
      Remove-AzDataFactoryV2Trigger -ResourceGroupName $using:deployResourceGroup -DataFactoryName $using:deployDataFactoryName -Name $_.Name -Force 
  }
}
# Remove pipelines
$pipelinesADF = Get-AzDataFactoryV2Pipeline -DataFactoryName $deployDataFactoryName -ResourceGroupName $deployResourceGroup
if($null -ne $pipelinesADF)
{ 
  $pipelinesADF | ForEach-Object -Parallel { 
      Remove-AzDataFactoryV2Pipeline -ResourceGroupName $using:deployResourceGroup -DataFactoryName $using:deployDataFactoryName -Name $_.Name -Force 
  }
}
# Try once more to remove the referenced pipelines which may not get deleted on the first execution
$pipelinesADF = Get-AzDataFactoryV2Pipeline -DataFactoryName $deployDataFactoryName -ResourceGroupName $deployResourceGroup
if($null -ne $pipelinesADF)
{ 
  $pipelinesADF | ForEach-Object -Parallel { 
      Remove-AzDataFactoryV2Pipeline -ResourceGroupName $using:deployResourceGroup -DataFactoryName $using:deployDataFactoryName -Name $_.Name -Force 
  }
}
# Remove datasets
$datasetsADF = Get-AzDataFactoryV2Dataset -DataFactoryName $deployDataFactoryName -ResourceGroupName $deployResourceGroup
if($null -ne $datasetsADF)
{ 
  $datasetsADF | ForEach-Object -Parallel { 
      Remove-AzDataFactoryV2Dataset -ResourceGroupName $using:deployResourceGroup -DataFactoryName $using:deployDataFactoryName -Name $_.Name -Force 
  }
}

## Prepare linked templates

# Retrieve storage account
$storageAccount = Get-AzStorageAccount -ResourceGroupName $storageResourceGroup `
                      -Name $storageAccountName

# Retrieve the context.
$storageContext = $storageAccount.Context

# Ensure container exists
$getContainer = Get-AzStorageContainer -Context $storageContext -Name $storageContainerName -ErrorAction:Ignore
If($null -eq $getContainer) {
    Write-Host "Creating '$storageContainerName' container"
    $paramNewAzStorageContainer = @{
    	Name	   = $storageContainerName
    	Context    = $storageContext
    	Permission = 'Off'
    }
    New-AzStorageContainer @paramNewAzStorageContainer -ErrorAction:Stop
} Else {
    Write-Host "'$storageContainerName' container exists"
}

# Upload partial ARM templates

$linkedARMTemplates=Get-ChildItem($linkedTemplateFolder) | Where-Object {$_.Name -match "ArmTemplate_\d+\.json"}

foreach($file in $linkedARMTemplates)
{
  Set-AzStorageBlobContent -File $file.FullName -Container $storageContainerName -Blob $file.Name -Context $storageContext -Force

  Write-Host "$($file.FullName) uploaded to $storageContainerName/$($file.Name)" -ForegroundColor Green
}

# Read parameters
$templateOptions= Get-Content $linkedTemplateParameterFile -Raw | ConvertFrom-Json

##### YOU CAN UPDATE ENVIRONMENT SETTINGS HERE ######

# Generate a sas token to pass to the arm templates
$sasToken=New-AzStorageContainerSASToken -Context $storageContext `
              -Name $storageContainerName -Permission r `
              -ExpiryTime ((Get-Date).AddHours($sasTokenTTL)) 

$templateOptions.parameters.containerUri.value= $storageAccount.PrimaryEndpoints.Blob + $storageContainerName
$templateOptions.parameters.containerSasToken.value=$sasToken

# Store the modified parameter file in order to inspect it later if needed
$newTemplateOptionsFile=$linkedTemplateParameterFile+".deployed.json"
$templateOptions | ConvertTo-Json -Depth 3 | Set-Content $newTemplateOptionsFile

# Deploy resources
New-AzResourceGroupDeployment -Name ADF_Linked_Template_Deployment -ResourceGroupName $deployResourceGroup `
             -TemplateFile $linkedTemplateMasterFile -TemplateParameterFile $newTemplateOptionsFile `
             -Verbose -Mode Incremental -Force

# Start all triggers after deploying        
$triggersADF = Get-AzDataFactoryV2Trigger -DataFactoryName $deployDataFactoryName -ResourceGroupName $deployResourceGroup
if($null -ne $triggersADF)
{ 
  $triggersADF | ForEach-Object -Parallel { 
      Start-AzDataFactoryV2Trigger -ResourceGroupName $using:deployResourceGroup -DataFactoryName $using:deployDataFactoryName -Name $_.Name -Force 
  }
}