How to configure Grafana Alloy to pull metrics from Azure and push them into Grafana Cloud for metrics / alerting consolidation

README.md

How to push Azure virtual machines metrics to Grafana Cloud

0. the alloy environment

jokeru@box:~# lsb_release -a
No LSB modules are available.
Distributor ID:	Ubuntu
Description:	Ubuntu 22.04.4 LTS
Release:	22.04
Codename:	jammy

jokeru@box:~# alloy --version
alloy, version v1.0.0 (branch: HEAD, revision: 1eef9b3ae)
  build user:       root@7197b65efb7f
  build date:       2024-04-05T20:54:07Z
  go version:       go1.22.1
  platform:         linux/amd64
  tags:             netgo,builtinassets,promtail_journal_enabled

1. setup the Azure auth for alloy

Azure cli login

az login --use-device-code

Create new custom role

az role definition create --role-definition '{
  "Name": "Metrics Reader",
  "Description": "Allows read access to Azure Metrics",
  "Actions": [
    "Microsoft.Insights/Metrics/read",
    "Microsoft.Compute/virtualMachines/read"
  ],
  "assignableScopes": [
    "/subscriptions/YOUR-SUBSCRIPTION-HERE/resourceGroups/YOUR-RESOURCE-GROUP-HERE"
  ]
}'

az role definition list --custom-role-only

Create new service principal

az ad sp create-for-rbac \
  --name "grafana-alloy-metrics" \
  --role "Metrics Reader" \
  --scopes "/subscriptions/YOUR-SUBSCRIPTION-HERE/resourceGroups/YOUR-RESOURCE-GROUP-HERE" \
  --years 5 \
  --json-auth

az ad sp list --display-name "grafana-alloy-metrics"

Make a note of the following 3 values, they will be used in the /etc/default/alloy file:

• tenantId
• clientId
• clientSecret

2. update /etc/default/alloy

3. update /etc/alloy/config.alloy

4. restart alloy service

etc_alloy_config.alloy

prometheus.remote_write "metrics_service" {
  endpoint {
    url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push"

    basic_auth {
      username = "YOUR-GRAFANA-CLOUD-PROM-USERNAME"
      password = "YOUR-GRAFANA-CLOUD-PROM-PASSWORD"
    }
  }
}

// Azure Exporter //

prometheus.exporter.azure "integrations_azure_exporter" {
  // https://grafana.com/docs/alloy/latest/reference/components/prometheus.exporter.azure/
  subscriptions    = ["YOUR-SUBSCRIPTION-HERE"]
  resource_type    = "Microsoft.Compute/virtualMachines"
  metric_namespace = "Microsoft.Compute/virtualMachines"
  metrics          = [
    "CPU Credits Remaining",
  ]
}

prometheus.scrape "integrations_azure_exporter" {
  targets    = prometheus.exporter.azure.integrations_azure_exporter.targets
  forward_to = [prometheus.relabel.integrations_azure_exporter.receiver]
  job_name   = "integrations/azure_exporter"
}

prometheus.relabel "integrations_azure_exporter" {
  forward_to = [prometheus.remote_write.metrics_service.receiver]

  // add standard instance label (consistency with node_exporter metrics)
  rule {
    source_labels = ["resourceName"]
    target_label = "instance"
  }
  // drop all labels except these two
  rule {
    regex  = "__name__|instance"
    action = "labelkeep"
  }
  // add standard job label
  rule {
    target_label = "job"
    replacement  = "integrations/azure_exporter"
  }
}

etc_default_alloy

## Path:
## Description: Grafana Alloy settings
## Type:        string
## Default:     ""
## ServiceRestart: alloy
#
# Command line options for Alloy.
#
# The configuration file holding the Alloy config.
CONFIG_FILE="/etc/alloy/config.alloy"

# User-defined arguments to pass to the run command.
CUSTOM_ARGS="--server.http.listen-addr 0.0.0.0:12345"

# Restart on system upgrade. Defaults to true.
RESTART_ON_UPGRADE=true

# Azure metrics auth
AZURE_TENANT_ID="YOUR-tenantId-HERE"
AZURE_CLIENT_ID="YOUR-clientId-HERE"
AZURE_CLIENT_SECRET="YOUR-clientSecret-HERE"