Virtual K8S

01-common.yaml

apiVersion: v1
kind: Secret
metadata:
  name: virtual
  namespace: federated
type: Opaque
data:
  kubeconfig: YXBpVmVyc2lvbjogdjEKa2luZDogQ29uZmlnCmNsdXN0ZXJzOgotIGNsdXN0ZXI6CiAgICBpbnNlY3VyZS1za2lwLXRscy12ZXJpZnk6IHRydWUKICAgIHNlcnZlcjogaHR0cHM6Ly9ub3QtdXNlZDoyNDQzCiAgbmFtZTogZHVtbXkKY29udGV4dHM6Ci0gY29udGV4dDoKICAgIGNsdXN0ZXI6IGR1bW15CiAgICB1c2VyOiBkdW1teQogIG5hbWU6IGR1bW15CmN1cnJlbnQtY29udGV4dDogZHVtbXkKdXNlcnM6Ci0gbmFtZTogZHVtbXkKICB1c2VyOgogICAgcGFzc3dvcmQ6IGFkbWluCiAgICB1c2VybmFtZTogYWRtaW4K

# The kubeconfig is more or less
# apiVersion: v1
# kind: Config
# clusters:
# - cluster:
#     insecure-skip-tls-verify: true
#     server: https://not-used:2443
#   name: dummy
# contexts:
# - context:
#     cluster: dummy
#     user: dummy
#   name: dummy
# current-context: dummy
# users:
# - name: dummy
#   user:
#     password: admin
#     username: admin

---

apiVersion: v1
kind: ConfigMap
metadata:
  name: basic-auth
  namespace: federated
data:
  basic_auth.csv: admin,admin,3,"cluster-admin,system:masters"

create.sh

#!/bin/bash

OPERATION=${1:-create}
CLUSTER_NUMBER=${2:-30}

cat <<EOF | kubectl ${OPERATION} -f -
apiVersion: v1
kind: Service
metadata:
  name: etcd
  namespace: federated
spec:
  selector:
    app: etcd
  ports:
  - port: 2379
---
apiVersion: v1
kind: Pod
metadata:
  labels:
    app: etcd
  name: etcd
  namespace: federated
spec:
  containers:
  - name: etcd
    image: quay.io/coreos/etcd:v3.3.2
    command:
    - etcd
    - -advertise-client-urls=http://127.0.0.1:2379
    - -listen-client-urls=http://0.0.0.0:2379
  terminationGracePeriodSeconds: 1
EOF

for i in $(seq 1 ${CLUSTER_NUMBER}); do
    cat <<EOF | kubectl ${OPERATION} -f -
apiVersion: clusterregistry.k8s.io/v1alpha1
kind: Cluster
metadata:
  name: virtual-${i}
  namespace: federated
  labels:
    type: virtual
spec:
  authInfo: {}
  kubernetesApiEndpoints:
    serverEndpoints:
    - clientCIDR: 0.0.0.0/0
      serverAddress: https://virtual-${i}.federated

---

apiVersion: core.federation.k8s.io/v1alpha1
kind: FederatedCluster
metadata:
  name: virtual-${i}
  namespace: federated
  labels:
    type: virtual
spec:
  clusterRef:
    name: virtual-${i}
  secretRef:
    name: virtual

---

apiVersion: v1
kind: Pod
metadata:
  labels:
    apiserver: virtual-${i}
  name: virtual-${i}
  namespace: federated
spec:
  containers:
  - name: kube-apiserver
    image: k8s.gcr.io/hyperkube:v1.13.1
    imagePullPolicy: IfNotPresent
    command:
    - /hyperkube
    - apiserver
    - --basic-auth-file=/srv/kubernetes/auth/basic_auth.csv
    - "--disable-admission-plugins=AlwaysAdmit,AlwaysDeny,AlwaysPullImages,DefaultStorageClass,DefaultTolerationSeconds,DenyEscalatingExec,DenyExecOnPrivileged,EventRateLimit,ExtendedResourceToleration,ImagePolicyWebhook,Initializers,LimitPodHardAntiAffinityTopology,LimitRanger,MutatingAdmissionWebhook,NamespaceAutoProvision,NamespaceExists,NamespaceLifecycle,NodeRestriction,OwnerReferencesPermissionEnforcement,PersistentVolumeClaimResize,PersistentVolumeLabel,PodNodeSelector,PodPreset,PodSecurityPolicy,PodTolerationRestriction,Priority,ResourceQuota,SecurityContextDeny,ServiceAccount,StorageObjectInUseProtection,ValidatingAdmissionWebhook"
    - "--runtime-config=admissionregistration.k8s.io/v1beta1=false,apps/v1=false,apps/v1beta1=false,apps/v1beta2=false,authentication.k8s.io/v1=false,authentication.k8s.io/v1beta1=false,authorization.k8s.io/v1=false,authorization.k8s.io/v1beta1=false,autoscaling/v1=false,autoscaling/v2beta1=false,autoscaling/v2beta2=false,batch/v1=false,batch/v1beta1=false,certificates.k8s.io/v1beta1=false,coordination.k8s.io/v1beta1=false,events.k8s.io/v1beta1=false,extensions/v1beta1=false,networking.k8s.io/v1=false,policy/v1beta1=false,rbac.authorization.k8s.io/v1=false,rbac.authorization.k8s.io/v1beta1=false,scheduling.k8s.io/v1beta1=false,storage.k8s.io/v1=false,storage.k8s.io/v1beta1=false"
    - --watch-cache=false
    - --enable-aggregator-routing=false
    - --enable-garbage-collector=false
    - --enable-bootstrap-token-auth=false
    - --enable-logs-handler=false
    - --enable-swagger-ui=false
    - --default-watch-cache-size=0
    - --etcd-servers=http://etcd:2379
    - --target-ram-mb=2
    - --secure-port=443
    - --endpoint-reconciler-type=none
    - --service-account-lookup=false
    - --profiling=false
    - --etcd-prefix=/registry-${i}
    - --v=2
    volumeMounts:
    - mountPath: /srv/kubernetes/auth
      name: basic-auth
  terminationGracePeriodSeconds: 1
  volumes:
  - configMap:
      name: basic-auth
    name: basic-auth

---

apiVersion: v1
kind: Service
metadata:
  name: virtual-${i}
  namespace: federated
spec:
  selector:
    apiserver: virtual-${i}
  ports:
  - port: 443
EOF

done

How to create many virtual K8S clusters.

First create the common resources first with

kubectl create namespace federated
kubectl create -f 01-common.yaml

and then create X amount of clusters with:

./create.sh create 70

For removal run:

./create.sh "delete --wait=false" 70

Great idea! Looking forward to reading your blog post!