Prepare two K8s clusters

Context east

Create cluster on GCP

$ gcloud container clusters create multi-cluster-demo --preemptible

Context west

Create cluster using kind/minikube/k3s

Deploy Ambassador

Using https://app.getambassador.io/initializer/ generate two set of configuration, one for the east cluster (GKE and L4 load-balancer), and another one for the west cluster (desktop), and deploy these YAML configs to the respective cluster.

$ EAST_GUID="xxx"
$ kubectl --context=east apply -f https://app.getambassador.io/initializer/yaml/${EAST_GUID}/crds && kubectl --context=east wait --for condition=established --timeout=90s crd -lproduct=aes
$ kubectl --context=east apply -f https://app.getambassador.io/initializer/yaml/${EAST_GUID}/install && kubectl --context=east wait -n ambassador deploy -lproduct=aes --for condition=available --timeout=90s
$ kubectl --context=east apply -f https://app.getambassador.io/initializer/yaml/${EAST_GUID}/configure

$ WEST_GUID="yyy"
$ kubectl --context=west apply -f https://app.getambassador.io/initializer/yaml/${WEST_GUID}/crds && kubectl --context=west wait --for condition=established --timeout=90s crd -lproduct=aes
$ kubectl --context=west apply -f https://app.getambassador.io/initializer/yaml/${WEST_GUID}/install && kubectl --context=west wait -n ambassador deploy -lproduct=aes --for condition=available --timeout=90s
$ kubectl --context=west apply -f https://app.getambassador.io/initializer/yaml/${WEST_GUID}/configure

You may additionally want to assign a DNS record to the east cluster load-balancer

$ AMBASSADOR_SERVICE_IP=$(kubectl --context=east get service -n ambassador ambassador -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
$ echo $AMBASSADOR_SERVICE_IP

Generate trust anchors CA

We need step CLI for that

$ step certificate create root.linkerd.cluster.local root.crt root.key --profile root-ca --no-password --insecure

Generate subordinate CA certificates

$ step certificate create identity.linkerd.cluster.local issuer.crt issuer.key --profile intermediate-ca --not-after 8760h --no-password --insecure --ca root.crt --ca-key root.key

Install linkerd

Make sure you have linkerd CLI installed first

https://linkerd.io/2.11/getting-started/

Proceed with installation

$ linkerd install \
  --identity-trust-anchors-file root.crt \
  --identity-issuer-certificate-file issuer.crt \
  --identity-issuer-key-file issuer.key \
  | tee \
    >(kubectl --context=west apply -f -) \
    >(kubectl --context=east apply -f -) > /dev/null

Check installation status

$ for ctx in west east; do
  echo "Checking cluster: ${ctx} .........\n"
  linkerd --context=${ctx} check || break
  echo "-------------\n"
done

Install multi-cluster component

We're going to use Ambassador as an API gateway instead of Linkerd Gateway, that's why we'll provide one additionall command-line parameter --gateway=false

$ for ctx in west east; do
  echo "Installing on cluster: ${ctx} ........."
  linkerd --context=${ctx} multicluster install --gateway=false | kubectl --context=${ctx} apply -f - || break
  echo "-------------\n"
done

Patch east cluster

Annotate Ambassador gateway

$ kubectl --context=east -n ambassador patch deploy ambassador -p='
spec:
  template:
    metadata:
      annotations:
        config.linkerd.io/enable-gateway: "true"
'

Patch Ambassador ports

$ kubectl --context=east -n ambassador patch svc ambassador --type='json' -p='[
  {"op": "add", "path": "/spec/ports/-", "value": {"name": "mc-gateway", "port": 4143}},
  {"op": "replace", "path": "/spec/ports/0", "value": {"name": "mc-probe", "port": 80, "targetPort": 8080}}
]'

Annotate Ambassador service

$ kubectl --context=east -n ambassador patch svc ambassador -p='
metadata:
  annotations:
    mirror.linkerd.io/gateway-identity: ambassador.ambassador.serviceaccount.identity.linkerd.cluster.local
    mirror.linkerd.io/multicluster-gateway: "true"
    mirror.linkerd.io/probe-path: -/ambassador/ready
    mirror.linkerd.io/probe-period: "3"
'

Check status

$ for ctx in west east; do
  echo "Checking cluster: ${ctx} ........."
  kubectl --context=${ctx} -n linkerd-multicluster rollout status deploy/linkerd-service-mirror || break
  echo "-------------\n"
done

$ for ctx in west east; do
  echo "Checking cluster: ${ctx} ........."
  linkerd --context=${ctx} check --multicluster || break
  echo "-------------\n"
done

$ kubectl --context=west -n linkerd-multicluster get all

Link two clusters together

$ linkerd --context=east multicluster link --cluster-name east | kubectl --context=west apply -f -

Check status

$ linkerd --context=west check --multicluster

Deploy sample app

$ for ctx in west east; do
  echo "Adding test services on cluster: ${ctx} ........."
  kubectl --context=${ctx} apply -k "github.com/linkerd/website/multicluster/${ctx}/"
  kubectl --context=${ctx} -n test rollout status deploy/podinfo || break
  echo "-------------\n"
done

Add mapping

$ cat <<'EOF' | kubectl --context=west apply -f -
---
apiVersion: getambassador.io/v2
kind: Mapping
metadata:
  name: frontend
spec:
  prefix: /
  service: frontend.test:8080
EOF

Redefine gateway

$ kubectl --context=east get svc -n test podinfo -o yaml | \
  linkerd multicluster export-service --gateway-name=ambassador --gateway-namespace=ambassador - | \
  kubectl --context=east apply -f -

$ kubectl --context=west -n test get endpoints podinfo-east -o 'custom-columns=ENDPOINT_IP:.subsets[*].addresses[*].ip'
$ kubectl --context=east -n ambassador get svc ambassador -o 'custom-columns=GATEWAY_IP:.status.loadBalancer.ingress[*].ip'

$ linkerd --context=west -n test viz stat --from deploy/frontend svc
$ linkerd --context=west -n test viz tap deploy/frontend | \
  grep "$(kubectl --context=east -n linkerd-multicluster get svc linkerd-gateway \
    -o "custom-columns=GATEWAY_IP:.status.loadBalancer.ingress[*].ip")"

Split traffic

$ cat <<'EOF' | kubectl --context=west apply -f -
apiVersion: split.smi-spec.io/v1alpha1
kind: TrafficSplit
metadata:
  name: podinfo
  namespace: test
spec:
  service: podinfo
  backends:
  - service: podinfo
    weight: 50
  - service: podinfo-east
    weight: 50
EOF

Check endpoints

apiVersion: v1
kind: Endpoints
metadata:
  name: podinfo-east
  namespace: test
subsets:
- addresses:
  - ip: $AMBASSADOR_SERVICE_IP

References

https://app.getambassador.io/initializer/

https://smallstep.com/cli/

https://buoyant.io/media/multi-cluster-with-linkerd-ambassador/

https://linkerd.io/2.11/tasks/multicluster/

avoidik/README.md