2 k3d clusters with 3 nodes each, using the same flat network

flat-network.sh

#!/usr/bin/env bash

set -xeuo pipefail

k3d_api_ready() {
    name=$1
    for i in {1..6} ; do
        if kubectl --context=k3d-$name cluster-info >/dev/null ; then return ; fi
        sleep 10
    done
    exit 1
}

k3d_dns_ready() {
    name=$1
    while [ $(kubectl --context=k3d-$name get po -n kube-system -l k8s-app=kube-dns -o json |jq '.items | length') = "0" ]; do sleep 1 ; done
    kubectl --context=k3d-$name wait pod --for=condition=ready \
        --namespace=kube-system --selector=k8s-app=kube-dns \
        --timeout=1m
}

create_cluster() {
    name=$1
    k3d cluster create -c "$name".yml --kubeconfig-update-default
    k3d_api_ready "$name"
    k3d_dns_ready "$name"
}

create_cluster source
create_cluster target

# add routes for each node in source to each node in target
kubectl --context=k3d-source get node -o json | \
jq -r '.items[] | .metadata.name + "\t" + .spec.podCIDR + "\t" + (.status.addresses[] | select(.type == "InternalIP") | .address)' | \
while IFS=$'\t' read -r sname scidr sip; do
    kubectl --context=k3d-target get node -o json | \
    jq -r '.items[] | .metadata.name + "\t" + .spec.podCIDR + "\t" + (.status.addresses[] | select(.type == "InternalIP") | .address)' | \
    while IFS=$'\t' read -r tname tcidr tip; do
        docker exec "${sname}" ip route add "${tcidr}" via "${tip}"
        docker exec "${tname}" ip route add "${scidr}" via "${sip}"
    done
done

source.yml

apiVersion: k3d.io/v1alpha4
kind: Simple
metadata:
  name: source
servers: 1
agents: 3
image: rancher/k3s:v1.26.4-k3s1
network: multiaz
options:
  k3d:
    disableLoadbalancer: true
  k3s:
    extraArgs:
      - arg: --disable=servicelb,traefik
        nodeFilters: [server:*]
      - arg: --cluster-cidr=10.22.0.0/16
        nodeFilters: [server:*]
      - arg: --service-cidr=10.246.0.0/16
        nodeFilters: [server:*]
      - arg: --debug
        nodeFilters: [server:*]
    nodeLabels:
      - label: topology.kubernetes.io/zone=zone-a
        nodeFilters: [agent:0]
      - label: topology.kubernetes.io/zone=zone-b
        nodeFilters: [agent:1]
      - label: topology.kubernetes.io/zone=zone-c
        nodeFilters: [agent:2]

target.yml

apiVersion: k3d.io/v1alpha4
kind: Simple
metadata:
  name: target
servers: 1
agents: 3
image: rancher/k3s:v1.26.4-k3s1
network: multiaz
options:
  k3d:
    disableLoadbalancer: true
  k3s:
    extraArgs:
      - arg: --disable=traefik
        nodeFilters: [server:*]
      - arg: --cluster-cidr=10.23.0.0/16
        nodeFilters: [server:*]
      - arg: --service-cidr=10.247.0.0/16
        nodeFilters: [server:*]
      - arg: --debug
        nodeFilters: [server:*]
    nodeLabels:
      - label: topology.kubernetes.io/zone=zone-a
        nodeFilters: [agent:0]
      - label: topology.kubernetes.io/zone=zone-b
        nodeFilters: [agent:1]
      - label: topology.kubernetes.io/zone=zone-c
        nodeFilters: [agent:2]