AKS Multicluster Linkerd setup

ca.crt

-----BEGIN CERTIFICATE-----
MIIBwDCCAWegAwIBAgIRAMV3cqmlAKmERWD5jbem3oQwCgYIKoZIzj0EAwIwKTEn
MCUGA1UEAxMeaWRlbnRpdHkubGlua2VyZC5jbHVzdGVyLmxvY2FsMB4XDTIwMDQx
MjE2NTY0N1oXDTMwMDQxMDE2NTY0N1owKTEnMCUGA1UEAxMeaWRlbnRpdHkubGlu
a2VyZC5jbHVzdGVyLmxvY2FsMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEwrlY
F+mIhjadG62LIhzukLt/9JUKpyOUoVWKYVAF87B6nE15yEKH7t/6fu3woDan/RCM
08wDXJEPa/DyTSnd06NwMG4wDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYB
Af8CAQEwHQYDVR0OBBYEFADWA55GU7kHnRE/SqBEudiZ2ML1MCkGA1UdEQQiMCCC
HmlkZW50aXR5LmxpbmtlcmQuY2x1c3Rlci5sb2NhbDAKBggqhkjOPQQDAgNHADBE
AiAyulh8SliUiPAeYRIYdTKqhmRi61+sSVmSXEKAQiKtRAIgd8zY9YLXxtjMNA6m
x3biAOuwLeUcSpbeXRgzbiqhPjc=
-----END CERTIFICATE-----

ca.key

-----BEGIN EC PRIVATE KEY-----
MHcCAQEEIIW/4W//fVN1cTckvwZRTo23pT/WoEe6BRdol4GoxveWoAoGCCqGSM49
AwEHoUQDQgAEwrlYF+mIhjadG62LIhzukLt/9JUKpyOUoVWKYVAF87B6nE15yEKH
7t/6fu3woDan/RCM08wDXJEPa/DyTSnd0w==
-----END EC PRIVATE KEY-----

issuer.crt

-----BEGIN CERTIFICATE-----
MIIBwTCCAWegAwIBAgIRAKH6iVemU4mgJhRSG/aVSnowCgYIKoZIzj0EAwIwKTEn
MCUGA1UEAxMeaWRlbnRpdHkubGlua2VyZC5jbHVzdGVyLmxvY2FsMB4XDTIwMDQx
MjE2NTgxMloXDTIxMDQxMjE2NTgxMlowKTEnMCUGA1UEAxMeaWRlbnRpdHkubGlu
a2VyZC5jbHVzdGVyLmxvY2FsMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEDIt3
YnCEIh/Xstzwxn77qJr/PcNNx5kieph+xE8AlHRed/9tj74BZ4YOZ96goWGFDglf
RaXauVhOqSSlpnfkaKNwMG4wDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYB
Af8CAQAwHQYDVR0OBBYEFPNhjDTlouKKKgQ72YlUkmiT956tMCkGA1UdEQQiMCCC
HmlkZW50aXR5LmxpbmtlcmQuY2x1c3Rlci5sb2NhbDAKBggqhkjOPQQDAgNIADBF
AiBJJnTPotWcB84QbNMlR+txhzyGOn4XcIWX/LmuXmT32QIhAKJPIggVJpRtEQd0
iVGvnVKeJfkuefo0wams7zEdyxxI
-----END CERTIFICATE-----

issuer.key

-----BEGIN EC PRIVATE KEY-----
MHcCAQEEIHPgZUJKQDGYZlB9PzUtoZwR3Z5ZUj+c3asaF2nm1JGQoAoGCCqGSM49
AwEHoUQDQgAEDIt3YnCEIh/Xstzwxn77qJr/PcNNx5kieph+xE8AlHRed/9tj74B
Z4YOZ96goWGFDglfRaXauVhOqSSlpnfkaA==
-----END EC PRIVATE KEY-----

main.tf

# Configure the Azure Provider
provider "azurerm" {
  # whilst the `version` attribute is optional, we recommend pinning to a given version of the Provider
  version = "=2.0.0"
  features {}
}

provider "azuread" {
  version = "=0.7.0"
}

# Create resource groups
resource "azurerm_resource_group" "uks-rg" {
  name     = "uks-cluster-rg"
  location = "uksouth"
}

resource "azurerm_resource_group" "ukw-rg" {
  name     = "ukw-cluster-rg"
  location = "ukwest"
}

// Virtual networks
resource "azurerm_virtual_network" "uks-vnet" {
  name                = "uks-vnet"
  location            = azurerm_resource_group.uks-rg.location
  resource_group_name = azurerm_resource_group.uks-rg.name
  address_space       = ["10.0.0.0/16"]

  subnet {
    name = "ClusterSubnet"
    address_prefix = "10.0.0.0/20"
  }
}

resource "azurerm_virtual_network" "ukw-vnet" {
  name                = "ukw-vnet"
  location            = azurerm_resource_group.ukw-rg.location
  resource_group_name = azurerm_resource_group.ukw-rg.name
  address_space       = ["10.1.0.0/16"]

  subnet {
    name = "ClusterSubnet"
    address_prefix = "10.1.0.0/20"
  }
}

// Peerings
resource "azurerm_virtual_network_peering" "uks2ukw-peering" {
  name                      = "uks2ukw-peering"
  resource_group_name       = azurerm_resource_group.uks-rg.name
  virtual_network_name      = azurerm_virtual_network.uks-vnet.name
  remote_virtual_network_id = azurerm_virtual_network.ukw-vnet.id
}

resource "azurerm_virtual_network_peering" "ukw2uks-peering" {
  name                      = "ukw2uks-peering"
  resource_group_name       = azurerm_resource_group.ukw-rg.name
  virtual_network_name      = azurerm_virtual_network.ukw-vnet.name
  remote_virtual_network_id = azurerm_virtual_network.uks-vnet.id
}

resource "azuread_application" "main" {
  name                       = "KubernetesCluster"
  available_to_other_tenants = false
}

resource "azuread_service_principal" "main" {
  application_id = azuread_application.main.application_id
}

resource "random_password" "main" {
  length  = 32
  special = false
}

resource "azuread_application_password" "main" {
  application_object_id  = azuread_application.main.id  
  value                  = random_password.main.result
  end_date               = "2299-12-31T00:00:00Z"
}

resource "azurerm_role_assignment" "uks-vnet-ra" {
  scope                = azurerm_virtual_network.uks-vnet.id
  role_definition_name = "Contributor"
  principal_id         = azuread_service_principal.main.id
}

resource "azurerm_role_assignment" "ukw-vnet-ra" {
  scope                = azurerm_virtual_network.ukw-vnet.id
  role_definition_name = "Contributor"
  principal_id         = azuread_service_principal.main.id
}

resource "azurerm_kubernetes_cluster" "uksk8s" {
  name = "uksk8s"
  location = azurerm_resource_group.uks-rg.location
  resource_group_name = azurerm_resource_group.uks-rg.name
  dns_prefix = "uksk8s-dns"
  kubernetes_version = "1.15.10"

  role_based_access_control {
    enabled = false
  }

  default_node_pool {
    name = "default"
    node_count = 1
    vm_size = "Standard_B2ms"
    type = "VirtualMachineScaleSets"
    vnet_subnet_id = "${azurerm_role_assignment.uks-vnet-ra.scope}/subnets/ClusterSubnet"
  }

  addon_profile {
    kube_dashboard {
      enabled = "true"
    }
  }

  network_profile {
    network_plugin = "azure"
    service_cidr = "172.20.0.0/16"
    dns_service_ip = "172.20.0.2"
    docker_bridge_cidr = "172.17.0.1/16"
  }

  service_principal {
    client_id = azuread_service_principal.main.application_id
    client_secret = azuread_application_password.main.value
  }

  depends_on = [
    azuread_service_principal.main,
    azuread_application_password.main
  ]
}

resource "azurerm_kubernetes_cluster" "ukwk8s" {
  name = "ukwk8s"
  location = azurerm_resource_group.ukw-rg.location
  resource_group_name = azurerm_resource_group.ukw-rg.name
  dns_prefix = "ukwk8s-dns"
  kubernetes_version = "1.15.7"

  role_based_access_control {
    enabled = false
  }

  default_node_pool {
    name = "default"
    node_count = 1
    vm_size = "Standard_B2ms"
    type = "VirtualMachineScaleSets"
    vnet_subnet_id = "${azurerm_role_assignment.ukw-vnet-ra.scope}/subnets/ClusterSubnet"
  }

  addon_profile {
    kube_dashboard {
      enabled = "true"
    }
  }

  network_profile {
    network_plugin = "azure"
    service_cidr = "172.21.0.0/16"
    dns_service_ip = "172.21.0.2"
    docker_bridge_cidr = "172.17.0.1/16"
  }

  service_principal {
    client_id = azuread_service_principal.main.application_id
    client_secret = azuread_application_password.main.value
  }

  depends_on = [
    azuread_service_principal.main,
    azuread_application_password.main
  ]
}

multicluster-gateway.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-configuration
  namespace: default
data:
  nginx.conf: |-
    events {
    }
    http {      
      include /etc/nginx/resolvers-data/resolvers.conf;
      server {
        location / {
          proxy_pass http://$http_host;
        }
      }
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: linkerd-gateway
  namespace: default
  labels:
    app: linkerd-gateway
spec:
  replicas: 1
  selector:
    matchLabels:
      app: linkerd-gateway
  template:
    metadata:
      labels:
        app: linkerd-gateway
    spec:
      volumes:
      - name: config
        configMap:
          name: nginx-configuration
      - name: resolvers-data
        emptyDir: {}    
      initContainers:
        - name: setup-resolvers
          image: busybox:1.28
          command: ["sh", "-c", "echo resolver $(awk 'BEGIN{ORS=\" \"} $1==\"nameserver\" {print $2}' /etc/resolv.conf)\";\" > /etc/nginx/resolvers-data/resolvers.conf"]
          volumeMounts:
            - name: resolvers-data
              mountPath: /etc/nginx/resolvers-data
      containers:
        - name: nginx
          image: nginx
          ports:
            - name: http
              containerPort: 80
          volumeMounts:
          - name: config
            mountPath: /etc/nginx
          - name: resolvers-data
            mountPath: /etc/nginx/resolvers-data      
---
apiVersion: v1
kind: Service
metadata:
  name: linkerd-gateway
  namespace: default
  annotations: 
    mirror.linkerd.io/gateway-identity: default.default.serviceaccount.identity.linkerd.cluster.local
    service.beta.kubernetes.io/azure-load-balancer-internal: "true"
spec:
  ports:
  - name: incoming-port
    port: 80
    protocol: TCP
  selector:
    app: linkerd-gateway
  type: LoadBalancer

remote-services.yaml

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: backend-one
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: backend-one
  template:
    metadata:
      labels:
        app: backend-one
    spec:
      containers:
      - name: backend-one
        image: buoyantio/bb:v0.0.5
        args:
        - terminus
        - "--h1-server-port=8888"
        - "--response-text=hello-remote-1"
        ports:
        - containerPort: 8888
---
apiVersion: v1
kind: Service
metadata:
  name: backend-one-svc
  namespace: default
spec:
  selector:
    app: backend-one
  ports:
  - name: http
    port: 8888
    targetPort: 8888
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: backend-two
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: backend-two
  template:
    metadata:
      labels:
        app: backend-two
    spec:
      containers:
      - name: backend-two
        image: buoyantio/bb:v0.0.5
        args:
        - terminus
        - "--h1-server-port=8888"
        - "--response-text=hello-remote-2"
        ports:
        - containerPort: 8888
---
apiVersion: v1
kind: Service
metadata:
  name: backend-two-svc
  namespace: default
spec:
  selector:
    app: backend-two
  ports:
  - name: http
    port: 8888
    targetPort: 8888    

setup.ps1

terraform init
terraform apply

az aks get-credentials -n uksk8s -g uks-cluster-rg 
az aks get-credentials -n ukwk8s -g ukw-cluster-rg 

linkerd install --context=uksk8s --identity-issuer-certificate-file .\issuer.crt --identity-issuer-key-file .\issuer.key --identity-trust-anchors-file .\ca.crt | kubectl --context=uksk8s apply -f -
linkerd install --context=ukwk8s --identity-issuer-certificate-file .\issuer.crt --identity-issuer-key-file .\issuer.key --identity-trust-anchors-file .\ca.crt | kubectl --context=ukwk8s apply -f -
linkerd check --context=uksk8s
linkerd check --context=ukwk8s

# Install service mirroring component
linkerd --context=uksk8s install-service-mirror --log-level=debug | kubectl --context=uksk8s apply -f -

# Create mirroring credentials on remote cluster
linkerd --context=ukwk8s cluster create-credentials | kubectl --context=ukwk8s apply -f -

# Enable mirroring
linkerd --context=ukwk8s cluster get-credentials --cluster-name=ukwk8s | kubectl --context=uksk8s apply -f -

# Install remote gateway
kubectl --context=ukwk8s apply -f .\multicluster-gateway.yaml
kubectl --context=ukwk8s get deploy linkerd-gateway -o yaml | linkerd --context=ukwk8s inject - | kubectl --context=ukwk8s apply -f -

# Install remote services
linkerd --context=ukwk8s inject .\remote-services.yaml | kubectl --context=ukwk8s apply -f -

# Export services
linkerd --context=ukwk8s cluster export-service --service-name=backend-one-svc --service-namespace=default --gateway-name=linkerd-gateway --gateway-ns=default
linkerd --context=ukwk8s cluster export-service --service-name=backend-two-svc --service-namespace=default --gateway-name=linkerd-gateway --gateway-ns=default

# Create test container
kubectl --context=uksk8s apply -f .\test-container.yaml
kubectl --context=uksk8s get deploy test -o yaml | linkerd --context=uksk8s inject - | kubectl --context=uksk8s apply -f -

test-container.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: test
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: test
  template:
    metadata:
      labels:
        app: test
    spec:
      containers:
      - name: test
        image: scrapinghub/httpbin:latest
        command:
        - sleep
        - "3600"
        resources:
          requests:
            cpu: 0.1
            memory: 200