AKS tf code using helm provider

readme.md

Creating the Azure Infrastructure using Terraform

We built using Terraform in 2 phases the tf infra:

1st command (the Kubernetes infrastructure):

terraform apply \
  -target=azurerm_resource_group.main \
  -target=azurerm_virtual_network.main \
  -target=azurerm_subnet.aks \
  -target=azurerm_log_analytics_workspace.main \
  -target=azurerm_recovery_services_vault.main \
  -target=azurerm_kubernetes_cluster.aks \
  -target=azurerm_backup_policy_vm.aks

So only apply the AKS cluster now. Skip the rest for now.

then:

az aks get-credentials \
  --resource-group aks-prod-rg \
  --name aks-prod-cluster \
  --admin \
  --file ~/.kube/aks-config

later: kubectl config get-contexts --kubeconfig ~/.kube/aks-config

export KUBECONFIG=~/.kube/aks-config

kubectl get nodes

2nd command (Installing ArgoCD & Traefik):

Please use: terraform apply

With this 2nd command, the other resources that are inside the phase2.tf file will be added (like Helm/Kubernetes ones), they depend on the AKS cluster being ready (used the phase1 command).

Finally: to visit Argo UI and Traefik Dashboard, you have to find the External-IP using kubectl get svc -n traefik, and later add in your /etc/hosts file like:

14.33.146.195 argocd.aks-prod.eastus.cloudapp.azure.com

14.33.146.195 traefik.aks-prod.eastus.cloudapp.azure.com

for production purposes you need to create a DNS record in Azure DNS.

Login to ArgoCD:

To login to ArgoCD, the default User is admin,

to find the password to login use this command:

This retrieves the auto-generated password:

kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 --decode; echo

or recreate a new bcrypt encrypted password using Htpasswd sudo apt install apache2-utils:

htpasswd -nbBC 10 "" "admin123" | tr -d ': ' 

later Patch the secret in the ArgoCD namespace:

kubectl -n argocd patch secret argocd-secret \
  -p '{"stringData": {"admin.password": "<YOUR-NEW-PASSWORD>"}}'

Finally, restart the ArgoCD Server Pod:

kubectl delete pod -n argocd -l app.kubernetes.io/name=argocd-server

Steps to Destroy all the infrastructure from Azure:

first delete manually the logs:

az resource delete \
  --resource-group aks-prod-rg \
  --name "ContainerInsights(aks-logs)" \
  --resource-type "Microsoft.OperationsManagement/solutions"

finally: terraform destroy

backend.tf

#terraform {

  # Uncomment and configure if you want to use remote state
  # backend "azurerm" {
  #   resource_group_name  = "terraform-state-rg"
  #   storage_account_name = "tfstate12345"
  #   container_name       = "tfstate"
  #   key                  = "aks-prod.terraform.tfstate"
  # }
# } 

main_phase1.tf

terraform {
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = ">= 3.65.0"
    }
  }
  required_version = ">= 1.0.0"
}

provider "azurerm" {
  features {}
  subscription_id = var.subscription_id
}

resource "azurerm_resource_group" "main" {
  name     = var.resource_group_name
  location = var.location
  tags = {
    Environment = var.environment
  }
}

resource "azurerm_log_analytics_workspace" "main" {
  name                = var.log_analytics_workspace_name
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
  sku                 = "PerGB2018"
  retention_in_days   = var.log_retention_in_days
}

resource "azurerm_recovery_services_vault" "main" {
  name                = var.backup_vault_name
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
  sku                 = "Standard"
  soft_delete_enabled = true
}

resource "azurerm_virtual_network" "main" {
  name                = "${var.cluster_name}-vnet"
  address_space       = ["10.0.0.0/16"]
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
}

resource "azurerm_subnet" "aks" {
  name                 = "${var.cluster_name}-subnet"
  resource_group_name  = azurerm_resource_group.main.name
  virtual_network_name = azurerm_virtual_network.main.name
  address_prefixes     = ["10.0.1.0/24"]
}

resource "azurerm_kubernetes_cluster" "aks" {
  name                = var.cluster_name
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
  dns_prefix          = var.dns_prefix

  default_node_pool {
    name           = "default"
    node_count     = var.node_count
    vm_size        = var.vm_size
    vnet_subnet_id = azurerm_subnet.aks.id
  }

  identity {
    type = "SystemAssigned"
  }

  network_profile {
    network_plugin     = "azure"
    network_policy     = var.enable_network_policy ? "azure" : null
    service_cidr       = "10.2.0.0/16"
    dns_service_ip     = "10.2.0.10"
  }

  azure_active_directory_role_based_access_control {
    tenant_id          = var.tenant_id
    azure_rbac_enabled = var.enable_azure_ad_integration
  }

  oms_agent {
    log_analytics_workspace_id = azurerm_log_analytics_workspace.main.id
  }

  tags = {
    Environment = var.environment
  }

  lifecycle {
    ignore_changes = [azure_active_directory_role_based_access_control]
  }
}

resource "azurerm_backup_policy_vm" "aks" {
  name                = "aks-backup-policy"
  resource_group_name = azurerm_resource_group.main.name
  recovery_vault_name = azurerm_recovery_services_vault.main.name

  backup {
    frequency = "Daily"
    time      = "23:00"
  }

  retention_daily {
    count = 7
  }

  retention_weekly {
    count    = 4
    weekdays = ["Sunday"]
  }

  retention_monthly {
    count    = 12
    weekdays = ["Sunday"]
    weeks    = ["First"]
  }
}

main_phase2.tf

provider "kubernetes" {
  config_path = "~/.kube/aks-config"
}

provider "helm" {
  kubernetes {
    config_path = "~/.kube/aks-config"
  }
}

resource "kubernetes_ingress_class" "traefik" {
  metadata {
    name = "traefik"
  }

  spec {
    controller = "traefik.io/ingress-controller"
  }
}

resource "kubernetes_namespace" "traefik" {
  metadata {
    name = var.traefik_namespace
  }
}

resource "helm_release" "traefik" {
  name       = "traefik"
  repository = "https://helm.traefik.io/traefik"
  chart      = "traefik"
  version    = var.traefik_version
  depends_on = [kubernetes_ingress_class.traefik]
  namespace  = kubernetes_namespace.traefik.metadata[0].name

  set {
    name  = "service.type"
    value = "LoadBalancer"
  }

  set {
    name  = "deployment.replicas"
    value = "2"
  }

  set {
    name  = "dashboard.enabled"
    value = "true"
  }

  set {
    name  = "dashboard.ingress.enabled"
    value = "true"
  }

  set {
    name  = "dashboard.ingress.annotations.kubernetes\\.io/ingress\\.class"
    value = "traefik"
  }

  set {
    name  = "dashboard.ingress.entrypoints[0]"
    value = "websecure"
  }

  set {
    name  = "dashboard.ingress.routes[0].match"
    value = "Host(`traefik.${var.dns_prefix}.${var.location}.cloudapp.azure.com`) && PathPrefix(`/dashboard`)"
  }

  set {
    name  = "dashboard.ingress.routes[0].kind"
    value = "Rule"
  }

  set {
    name  = "dashboard.ingress.annotations.traefik\\.ingress\\.kubernetes\\.io/router\\.tls"
    value = "true"
  }
}

resource "kubernetes_namespace" "argocd" {
  metadata {
    name = var.argocd_namespace
  }
}

resource "helm_release" "argocd" {
  name       = "argocd"
  repository = "https://argoproj.github.io/argo-helm"
  chart      = "argo-cd"
  version    = var.argocd_version
  depends_on = [kubernetes_ingress_class.traefik]
  namespace  = kubernetes_namespace.argocd.metadata[0].name

  set {
    name  = "server.extraArgs[0]"
    value = "--insecure"
  }
  set {
    name  = "server.service.type"
    value = "ClusterIP"
  }
  set {
    name  = "server.ingress.enabled"
    value = "true"
  }
  set {
    name  = "server.ingress.hosts[0]"
    value = "argocd.${var.dns_prefix}.${var.location}.cloudapp.azure.com"
  }
  set {
    name  = "server.ingress.ingressClassName"
    value = "traefik"
  }
  set {
    name  = "server.ingress.annotations.kubernetes\\.io/ingress\\.class"
    value = "traefik"
  }
  set {
    name  = "server.ingress.annotations.traefik\\.ingress\\.kubernetes\\.io/router\\.entrypoints"
    value = "websecure"
  }
  set {
    name  = "server.ingress.annotations.traefik\\.ingress\\.kubernetes\\.io/router\\.tls"
    value = "true"
  }
  set {
    name  = "redis.enabled"
    value = "true"
  }
  set {
    name  = "controller.replicas"
    value = "2"
  }
  set {
    name  = "repoServer.replicas"
    value = "2"
  }

  lifecycle {
    prevent_destroy = false
  }
}

resource "kubernetes_config_map" "argocd_rbac" {
  metadata {
    name = "argocd-rbac-custom"
    namespace = var.argocd_namespace
    labels = {
      "app.kubernetes.io/part-of" = "argocd"
    }
  }

  data = {
    "policy.csv" = <<-EOT
p, role:org-admin, applications, *, */*, allow
p, role:org-admin, clusters, get, *, allow
p, role:org-admin, repositories, get, *, allow
p, role:org-admin, repositories, create, *, allow
p, role:org-admin, repositories, update, *, allow
p, role:org-admin, repositories, delete, *, allow
g, org-admin, role:org-admin
EOT
  }
}

resource "null_resource" "patch_argocd_rbac_env" {
  depends_on = [helm_release.argocd, kubernetes_config_map.argocd_rbac]

  provisioner "local-exec" {
    command = <<-EOT
      kubectl -n ${var.argocd_namespace} set env deployment/argocd-server \
        ARGOCD_RBAC_POLICY_CSV="$(kubectl get configmap argocd-rbac-custom -n ${var.argocd_namespace} -o jsonpath='{.data.policy\\.csv}')"
    EOT
  }
}

# Patching the admin password for ArgoCD
# resource "null_resource" "patch_argocd_admin_password" {
#   depends_on = [helm_release.argocd]

#   provisioner "local-exec" {
#     command = <<-EOT
#       kubectl -n ${var.argocd_namespace} patch secret argocd-secret \
#         -p '{"stringData": {"admin.password": "$2a$10$rRyBsGSHK6.uc8fntPwVIuLVHgsAhAX7TcdrqW/RADU0uh7CaChLa"}}'
#     EOT
#   }
# }
# resource "null_resource" "restart_argocd_server" {
#   depends_on = [null_resource.patch_argocd_admin_password]

#   provisioner "local-exec" {
#     command = "kubectl -n ${var.argocd_namespace} delete pod -l app.kubernetes.io/name=argocd-server"
#   }
# }

NOT_USE_main.tf

#not use this! instead exist now 2 files as phases!
terraform {
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = ">= 3.65.0"
    }
    helm = {
      source  = "hashicorp/helm"
      version = "~> 2.0"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.0"
    }
  }
  required_version = ">= 1.0.0"
}

provider "azurerm" {
  features {}
  subscription_id = var.subscription_id
}

provider "helm" {
  kubernetes {
    host                   = azurerm_kubernetes_cluster.aks.kube_config[0].host
    client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config[0].client_certificate)
    client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config[0].client_key)
    cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config[0].cluster_ca_certificate)
  }
}

provider "kubernetes" {
  host                   = azurerm_kubernetes_cluster.aks.kube_config[0].host
  client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config[0].client_certificate)
  client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config[0].client_key)
  cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config[0].cluster_ca_certificate)
} 

# Resource Group
resource "azurerm_resource_group" "main" {
  name     = var.resource_group_name
  location = var.location
  tags = {
    Environment = var.environment
  }
}

# Log Analytics Workspace for Monitoring
resource "azurerm_log_analytics_workspace" "main" {
  name                = var.log_analytics_workspace_name
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
  sku                 = "PerGB2018"
  retention_in_days   = var.log_retention_in_days
}

# Backup Vault
resource "azurerm_recovery_services_vault" "main" {
  name                = var.backup_vault_name
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
  sku                 = "Standard"
  soft_delete_enabled = true
}

# Virtual Network
resource "azurerm_virtual_network" "main" {
  name                = "${var.cluster_name}-vnet"
  address_space       = ["10.0.0.0/16"]
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
}

# Subnet
resource "azurerm_subnet" "aks" {
  name                 = "${var.cluster_name}-subnet"
  resource_group_name  = azurerm_resource_group.main.name
  virtual_network_name = azurerm_virtual_network.main.name
  address_prefixes     = ["10.0.1.0/24"]
}

# AKS Cluster
resource "azurerm_kubernetes_cluster" "aks" {
  name                = var.cluster_name
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
  dns_prefix          = var.dns_prefix

  default_node_pool {
    name       = "default"
    node_count = var.node_count
    vm_size    = var.vm_size
    vnet_subnet_id = azurerm_subnet.aks.id
  }

  identity {
    type = "SystemAssigned"
  }

  network_profile {
    network_plugin = "azure"
    network_policy = var.enable_network_policy ? "azure" : null
    service_cidr      = "10.2.0.0/16"
    dns_service_ip    = "10.2.0.10"
  }

  # Azure AD Integration
  azure_active_directory_role_based_access_control {
    tenant_id          = var.tenant_id
    azure_rbac_enabled = var.enable_azure_ad_integration
  }

  # Monitoring
  oms_agent {
    log_analytics_workspace_id = azurerm_log_analytics_workspace.main.id
  }

  tags = {
    Environment = var.environment
  }

  lifecycle {
    ignore_changes = [azure_active_directory_role_based_access_control]
  }
}

# Traefik Namespace
resource "kubernetes_namespace" "traefik" {
  metadata {
    name = var.traefik_namespace
  }
}

# Traefik Helm Release
resource "helm_release" "traefik" {
  name       = "traefik"
  repository = "https://helm.traefik.io/traefik"
  chart      = "traefik"
  version    = var.traefik_version
  namespace  = kubernetes_namespace.traefik.metadata[0].name

  set {
    name  = "deployment.replicas"
    value = "2"
  }

  set {
    name  = "globalArguments[0]"
    value = "--global.checknewversion=false"
  }

  set {
    name  = "globalArguments[1]"
    value = "--global.sendanonymoususage=false"
  }

  set {
    name  = "providers.kubernetesIngress.publishedService.enabled"
    value = "true"
  }

  set {
    name  = "ports.web.redirectTo"
    value = "websecure"
  }

  set {
    name  = "ports.websecure.tls.enabled"
    value = "true"
  }

  set {
    name  = "service.type"
    value = "LoadBalancer"
  }

  set {
    name  = "dashboard.enabled"
    value = "true"
  }

  set {
    name  = "dashboard.ingress.enabled"
    value = "true"
  }

  set {
    name  = "dashboard.ingress.annotations.kubernetes\\.io/ingress\\.class"
    value = "traefik"
  }

  set {
    name  = "dashboard.ingress.entrypoints"
    value = "websecure"
  }

  set {
    name  = "dashboard.ingress.routes[0].match"
    value = "Host(`traefik.${var.dns_prefix}.${var.location}.cloudapp.azure.com`)"
  }
}

# Argo CD Namespace
resource "kubernetes_namespace" "argocd" {
  metadata {
    name = var.argocd_namespace
  }
}

# Argo CD Helm Release
resource "helm_release" "argocd" {
  name       = "argocd"
  repository = "https://argoproj.github.io/argo-helm"
  chart      = "argo-cd"
  version    = var.argocd_version
  namespace  = kubernetes_namespace.argocd.metadata[0].name

  # Configure Argo CD to run inside the cluster
  set {
    name  = "server.extraArgs[0]"
    value = "--insecure"
  }

  # Configure the server to be accessible within the cluster
  set {
    name  = "server.service.type"
    value = "ClusterIP"
  }

  # Enable ingress for external access
  set {
    name  = "server.ingress.enabled"
    value = "true"
  }

  set {
    name  = "server.ingress.hosts[0]"
    value = "argocd.${var.dns_prefix}.${var.location}.cloudapp.azure.com"
  }

  set {
    name  = "server.ingress.ingressClassName"
    value = "traefik"
  }

  set {
    name  = "server.ingress.annotations.kubernetes\\.io/ingress\\.class"
    value = "traefik"
  }

  set {
    name  = "server.ingress.annotations.traefik\\.ingress\\.kubernetes\\.io/router\\.entrypoints"
    value = "websecure"
  }

  set {
    name  = "server.ingress.annotations.traefik\\.ingress\\.kubernetes\\.io/router\\.tls"
    value = "true"
  }

  # Configure Redis for Argo CD (runs inside the cluster)
  set {
    name  = "redis.enabled"
    value = "true"
  }

  # Configure the application controller
  set {
    name  = "controller.replicas"
    value = "2"
  }

  # Configure the repo server
  set {
    name  = "repoServer.replicas"
    value = "2"
  }

  # Set admin password
  set {
    name  = "configs.secret.argocdServerAdminPassword"
    value = "$2a$10$rRyBsGSHK6.uc8fntPwVIuLVHgsAhAX7TcdrqW/RADU0uh7CaChLa" # "admin" password
  }

  # Configure RBAC
  set {
    name  = "server.rbacConfig.policy.default"
    value = "role:readonly"
  }

  set {
    name  = "server.rbacConfig.policy.csv"
    value = <<-EOT
      p, role:org-admin, applications, *, */*, allow
      p, role:org-admin, clusters, get, *, allow
      p, role:org-admin, repositories, get, *, allow
      p, role:org-admin, repositories, create, *, allow
      p, role:org-admin, repositories, update, *, allow
      p, role:org-admin, repositories, delete, *, allow
      g, org-admin, role:org-admin
    EOT
  }

  lifecycle {
    prevent_destroy = false
  }
}

# Backup Policy for AKS
resource "azurerm_backup_policy_vm" "aks" {
  name                = "aks-backup-policy"
  resource_group_name = azurerm_resource_group.main.name
  recovery_vault_name = azurerm_recovery_services_vault.main.name

  backup {
    frequency = "Daily"
    time      = "23:00"
  }

  retention_daily {
    count = 7
  }

  retention_weekly {
    count    = 4
    weekdays = ["Sunday"]
  }

  retention_monthly {
    count    = 12
    weekdays = ["Sunday"]
    weeks    = ["First"]
  }
}

outputs.tf

output "resource_group_name" {
  value = azurerm_resource_group.main.name
}

output "cluster_name" {
  value = azurerm_kubernetes_cluster.aks.name
}

output "kube_config" {
  value     = azurerm_kubernetes_cluster.aks.kube_config_raw
  sensitive = true
}

output "host" {
  value = azurerm_kubernetes_cluster.aks.kube_config[0].host
  sensitive = true
}

output "traefik_namespace" {
  value = kubernetes_namespace.traefik.metadata[0].name
}

output "traefik_version" {
  value = helm_release.traefik.version
}

output "argocd_namespace" {
  value = kubernetes_namespace.argocd.metadata[0].name
}

output "argocd_version" {
  value = helm_release.argocd.version
}

output "argocd_url" {
  value = "https://argocd.${var.dns_prefix}.${var.location}.cloudapp.azure.com"
}

output "argocd_admin_password" {
  value     = "admin"
  sensitive = true
}

# Monitoring outputs
output "log_analytics_workspace_id" {
  value = azurerm_log_analytics_workspace.main.workspace_id
}

output "log_analytics_workspace_name" {
  value = azurerm_log_analytics_workspace.main.name
}

# Backup outputs
output "backup_vault_name" {
  value = azurerm_recovery_services_vault.main.name
}

output "backup_policy_name" {
  value = azurerm_backup_policy_vm.aks.name
} 

terraform.tfvars

tenant_id = "azure-xxxxxxxxxxxxxxxxxxxxx"
subscription_id = "azure-xxxxxxxxxxxxxxx"

variables.tf

variable "resource_group_name" {
  description = "Name of the resource group"
  type        = string
  default     = "aks-prod-rg"
}

variable "location" {
  description = "Azure region where resources will be created"
  type        = string
  default     = "eastus"
}

variable "cluster_name" {
  description = "Name of the AKS cluster"
  type        = string
  default     = "aks-prod-cluster"
}

variable "dns_prefix" {
  description = "DNS prefix for the AKS cluster"
  type        = string
  default     = "aks-prod"
}

variable "node_count" {
  description = "Number of nodes in the default node pool"
  type        = number
  default     = 3
}

variable "vm_size" {
  description = "Size of the VMs in the node pool"
  type        = string
  default     = "Standard_D2s_v3"
}

variable "traefik_version" {
  description = "Version of Traefik to install"
  type        = string
  default     = "10.24.0"
}

variable "traefik_namespace" {
  description = "Namespace for Traefik installation"
  type        = string
  default     = "traefik"
}

variable "argocd_version" {
  description = "Version of Argo CD to install"
  type        = string
  default     = "5.46.8"
}

variable "argocd_namespace" {
  description = "Namespace for Argo CD installation"
  type        = string
  default     = "argocd"
}

variable "environment" {
  description = "Environment name (e.g., prod, staging)"
  type        = string
  default     = "production"
}

# Monitoring and Logging
variable "log_analytics_workspace_name" {
  description = "Name of the Log Analytics workspace"
  type        = string
  default     = "aks-logs"
}

variable "log_retention_in_days" {
  description = "Number of days to retain logs"
  type        = number
  default     = 30
}

# Backup
variable "backup_vault_name" {
  description = "Name of the Backup Vault"
  type        = string
  default     = "aks-backup-vault"
}

# Security
variable "enable_azure_ad_integration" {
  description = "Enable Azure AD integration for AKS"
  type        = bool
  default     = true
}

variable "enable_network_policy" {
  description = "Enable network policy for AKS"
  type        = bool
  default     = true
} 

variable "tenant_id" {
  description = "Azure Active Directory tenant ID"
  type        = string
}

variable "subscription_id" {
  description = "Azure Subscription ID"
  type        = string
}