my_mlops_stacks-bundle-cicd.yml

# This Azure Pipeline validates and deploys bundle config (ML resource config and more)
# defined under my_mlops_stacks/resources/*
# and my_mlops_stacks/databricks.yml.
# The bundle is validated (CI) upon making a PR against the main_mlops_stack branch.
# Bundle resources defined for staging are deployed when a PR is merged into the main_mlops_stack branch.
# Bundle resources defined for prod are deployed when a PR is merged into the release_mlops_stack branch.

trigger:
  branches:
    include:
      - dev_mlops_stack
      - main_mlops_stack
      - release_mlops_stack
  paths:
    include:
      - my_mlops_stacks/*

variables:
  - name: workingDirectory
    value: my_mlops_stacks
  - group: my-mlops-stacks variable group

stages:
# Run BundleCI stage upon making a PR against the main_mlops_stack branch
- stage: BundleCI
  displayName: 'Bundle validation for my-mlops-stacks'
  # Trigger BundleCI stage on PR against the default branch, and not on pushes to other branches
  condition: |
    and(
      not(eq(variables['Build.Reason'], 'IndividualCI')),
      eq(variables['Build.Reason'], 'PullRequest'),
      eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/main_mlops_stack')
    )

  jobs:
  - job: StagingBundleCI
    displayName: 'Staging bundle validation for my-mlops-stacks'

    pool:
      vmImage: 'ubuntu-latest'

    steps:
    - script: env | sort
      displayName: 'Environment / Context'

    - checkout: self
      displayName: 'Checkout & Build.Reason: $(Build.Reason) & Build.SourceBranchName: $(Build.SourceBranchName)'
      persistCredentials: true
      clean: true

    # Install Databricks CLI
    - script: |
        curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.212.2/install.sh | sh
      displayName: 'Install Databricks CLI'

    # Validate bundle to be deployed to the staging workspace
    - script: |
        databricks bundle validate -t staging
      workingDirectory: $(workingDirectory)
      displayName: 'Validate bundle for staging'
      env:
        ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
        ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)

  - job: prodBundleCI
    displayName: 'Prod bundle validation for my-mlops-stacks'
    dependsOn: []   # Removes the implicit dependency on previous job and force prodBundleCI job to run in parallel

    steps:
    - script: env | sort
      displayName: 'Environment / Context'

    - checkout: self
      displayName: 'Checkout & Build.Reason: $(Build.Reason) & Build.SourceBranchName: $(Build.SourceBranchName)'
      persistCredentials: true
      clean: true

    # Install Databricks CLI
    - script: |
        curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.212.2/install.sh | sh
      displayName: 'Install Databricks CLI'

    # Validate bundle to be deployed to the prod workspace
    - script: |
        databricks bundle validate -t prod
      workingDirectory: $(workingDirectory)
      displayName: 'Validate bundle for prod'
      env:
        ARM_TENANT_ID: $(PROD_AZURE_SP_TENANT_ID)
        ARM_CLIENT_ID: $(PROD_AZURE_SP_APPLICATION_ID)
        ARM_CLIENT_SECRET: $(PROD_AZURE_SP_CLIENT_SECRET)

# Run StagingBundleCD stage after successfully merging into the dev_mlops_stack branch
- stage: DevBundleCD
  displayName: 'Dev bundle deployment for my-mlops-stacks'
  # Trigger deployment of bundle resources when PRs are merged into the dev_mlops_stack branch
  condition: |
    and(
      eq(variables['Build.SourceBranch'], 'refs/heads/dev_mlops_stack'),
      not(eq(variables['Build.Reason'], 'PullRequest'))
    )

  jobs:
  - job: StagingBundleCD
    displayName: 'Bundle Deployment for my-mlops-stacks Dev'

    pool:
      vmImage: 'ubuntu-latest'

    steps:
    - script: env | sort
      displayName: 'Environment / Context'

    - checkout: self
      displayName: 'Checkout & Build.Reason: $(Build.Reason) & Build.SourceBranchName: $(Build.SourceBranchName)'
      persistCredentials: true
      clean: true

      # Install Databricks CLI
    - script: |
        curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.212.2/install.sh | sh
      displayName: 'Install Databricks CLI'

    # Validate bundle to be deployed to the Dev workspace
    - script: |
        databricks bundle validate -t dev
      workingDirectory: $(workingDirectory)
      displayName: 'Validate bundle for Dev'
      env:
        ARM_TENANT_ID: $(DEV_AZURE_SP_TENANT_ID)
        ARM_CLIENT_ID: $(DEV_AZURE_SP_APPLICATION_ID)
        ARM_CLIENT_SECRET: $(DEV_AZURE_SP_CLIENT_SECRET)

    # Deploy bundle to Dev workspace
    - script: |
        databricks bundle deploy -t dev
      workingDirectory: $(workingDirectory)
      displayName: 'Deploy bundle to dev'
      env:
        ARM_TENANT_ID: $(DEV_AZURE_SP_TENANT_ID)
        ARM_CLIENT_ID: $(DEV_AZURE_SP_APPLICATION_ID)
        ARM_CLIENT_SECRET: $(DEV_AZURE_SP_CLIENT_SECRET)

# Run StagingBundleCD stage after successfully merging into the main_mlops_stack branch
- stage: StagingBundleCD
  displayName: 'Staging bundle deployment for my-mlops-stacks'
  # Trigger deployment of bundle resources when PRs are merged into the main_mlops_stack branch
  condition: |
    and(
      eq(variables['Build.SourceBranch'], 'refs/heads/main_mlops_stack'),
      not(eq(variables['Build.Reason'], 'PullRequest'))
    )

  jobs:
  - job: StagingBundleCD
    displayName: 'Bundle Deployment for my-mlops-stacks Staging'

    pool:
      vmImage: 'ubuntu-latest'

    steps:
    - script: env | sort
      displayName: 'Environment / Context'

    - checkout: self
      displayName: 'Checkout & Build.Reason: $(Build.Reason) & Build.SourceBranchName: $(Build.SourceBranchName)'
      persistCredentials: true
      clean: true

      # Install Databricks CLI
    - script: |
        curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.212.2/install.sh | sh
      displayName: 'Install Databricks CLI'

    # Validate bundle to be deployed to the Staging workspace
    - script: |
        databricks bundle validate -t staging
      workingDirectory: $(workingDirectory)
      displayName: 'Validate bundle for staging'
      env:
        ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
        ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)

    # Deploy bundle to Staging workspace
    - script: |
        databricks bundle deploy -t staging
      workingDirectory: $(workingDirectory)
      displayName: 'Deploy bundle to staging'
      env:
        ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
        ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)

# Run prod bundle CD stage after successfully merging into the release_mlops_stack branch
- stage: prodBundleCD
  displayName: 'Prod bundle deployment for my-mlops-stacks'
  # Trigger deployment of Bundle resources when PRs are merged into the release_mlops_stack branch
  condition: |
    and(
      eq(variables['Build.SourceBranch'], 'refs/heads/release_mlops_stack'),
      not(eq(variables['Build.Reason'], 'PullRequest'))
    )

  jobs:
  - job: prodBundleCD
    displayName: 'Bundle deployment for my-mlops-stacks prod'

    pool:
      vmImage: 'ubuntu-latest'

    steps:
    - script: env | sort
      displayName: 'Environment / Context'

    - checkout: self
      displayName: 'Checkout & Build.Reason: $(Build.Reason) & Build.SourceBranchName: $(Build.SourceBranchName)'
      persistCredentials: true
      clean: true

      # Install Databricks CLI
    - script: |
        curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.212.2/install.sh | sh
      displayName: 'Install Databricks CLI'

    # Validate bundle to be deployed to the prod workspace
    - script: |
        databricks bundle validate -t prod
      workingDirectory: $(workingDirectory)
      displayName: 'Validate bundle for prod'
      env:
        ARM_TENANT_ID: $(PROD_AZURE_SP_TENANT_ID)
        ARM_CLIENT_ID: $(PROD_AZURE_SP_APPLICATION_ID)
        ARM_CLIENT_SECRET: $(PROD_AZURE_SP_CLIENT_SECRET)


    # Deploy bundle to prod workspace
    - script: |
        databricks bundle deploy -t prod
      workingDirectory: $(workingDirectory)
      displayName: 'Deploy bundle to prod'
      env:
        ARM_TENANT_ID: $(PROD_AZURE_SP_TENANT_ID)
        ARM_CLIENT_ID: $(PROD_AZURE_SP_APPLICATION_ID)
        ARM_CLIENT_SECRET: $(PROD_AZURE_SP_CLIENT_SECRET)