Bind Kubernetes Pod To Static Service IP Using socat (Rook & Ceph Example)

binding-service-ips-in-k8s-pods.md

This workaround solves many issues where software in a Kubernetes pod expects a static, rarely changed IP address. Since we bind to a Kubernetes Service IP, it comes with many benefits such as automatic DNS entries within the cluster, health checking automation, and abstraction of network details. In fact these features of Services are used in the pod examples here to avoid hard-coding IP addresses and ports into the pod definitions.

TODO: More details.

Results

    cluster 9ece8946-1b9d-4bb5-8d51-34c968119795
     health HEALTH_OK
     monmap e38: 3 mons at {mon0=10.3.0.83:6790/0,mon1=10.3.0.97:6790/0,mon2=10.3.0.189:6790/0}
            election epoch 332, quorum 0,1,2 mon0,mon1,mon2
        mgr no daemons active
     osdmap e2786: 22 osds: 22 up, 22 in
            flags sortbitwise,require_jewel_osds,require_kraken_osds
      pgmap v1347542: 2100 pgs, 2 pools, 17444 MB data, 4924 objects
            52137 MB used, 8292 GB / 8343 GB avail
                2100 active+clean
  client io 1831 kB/s wr, 0 op/s rd, 2 op/s wr

NAME                           READY     STATUS    RESTARTS   AGE       IP             NODE           LABELS
po/mon0                        2/2       Running   0          1h        10.2.247.43    node1.zbrbdl   app=mon,mon_cluster=rookcluster,name=mon0
po/mon1                        2/2       Running   0          1h        10.2.136.148   node2.zbrbdl   app=mon,mon_cluster=rookcluster,name=mon1
po/mon2                        2/2       Running   0          38m       10.2.6.23      node3.zbrbdl   app=mon,mon_cluster=rookcluster,name=mon2
po/osd-node1-l84ql             1/1       Running   0          1d        10.2.247.39    node1.zbrbdl   app=osd,node=node1,rook_cluster=rookcluster
po/osd-node2-4v4xn             1/1       Running   0          1d        10.2.136.172   node2.zbrbdl   app=osd,node=node2,rook_cluster=rookcluster
po/osd-node3-2zf1s             1/1       Running   0          1d        10.2.6.1       node3.zbrbdl   app=osd,node=node3,rook_cluster=rookcluster
po/osd-node4-0n11x             1/1       Running   0          1d        10.2.186.45    node4.zbrbdl   app=osd,node=node4,rook_cluster=rookcluster
po/rook-api-4085589939-bw0td   1/1       Running   0          1d        10.2.247.20    node1.zbrbdl   app=rook-api,pod-template-hash=4085589939,rook_cluster=rookcluster
po/rook-tools                  1/1       Running   0          1d        10.2.247.3     node1.zbrbdl   <none>

NAME            CLUSTER-IP   EXTERNAL-IP   PORT(S)    AGE       SELECTOR                                    LABELS
svc/mon0        10.3.0.218   <none>        6790/TCP   1h        app=mon,mon_cluster=rookcluster,name=mon0   app=mon,mon_cluster=rookcluster,name=mon0
svc/mon1        10.3.0.97    <none>        6790/TCP   3d        app=mon,mon_cluster=rookcluster,name=mon1   app=mon,mon_cluster=rookcluster,name=mon1
svc/mon2        10.3.0.189   <none>        6790/TCP   3d        app=mon,mon_cluster=rookcluster,name=mon2   app=mon,mon_cluster=rookcluster,name=mon2
svc/rook-api    10.3.0.213   <none>        8124/TCP   36d       app=rook-api,rook_cluster=rookcluster       app=rook-api,rook_cluster=rookcluster

rook-mon-services.yaml

apiVersion: v1
kind: Service
metadata:
  labels:
    app: mon
    mon_cluster: rookcluster
    name: mon0
  name: mon0
spec:
  type: ClusterIP
  ports:
  - name: rook-mon
    port: 6790
    targetPort: rook-mon
  selector:
    app: mon
    mon_cluster: rookcluster
    name: mon0
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: mon
    mon_cluster: rookcluster
    name: mon1
  name: mon1
spec:
  type: ClusterIP
  ports:
  - name: rook-mon
    port: 6790
    targetPort: rook-mon
  selector:
    app: mon
    mon_cluster: rookcluster
    name: mon1
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: mon
    mon_cluster: rookcluster
    name: mon2
  name: mon2
spec:
  type: ClusterIP
  ports:
  - name: rook-mon
    port: 6790
    targetPort: rook-mon
  selector:
    app: mon
    mon_cluster: rookcluster
    name: mon2

rook-mon0-pod.yaml

apiVersion: v1
kind: Pod
metadata:
  annotations:
    rook_version: v0.3.1
    scheduler.alpha.kubernetes.io/affinity: '{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"mon_cluster":"rookcluster"}},"namespaces":null,"topologyKey":"kubernetes.io/hostname"}]}}'
    pod.beta.kubernetes.io/init-containers: '
    [{
      "name": "ip-setup",
      "image": "busybox",
      "command": [
        "sh",
        "-xc", "
          ip addr add dev lo $(MON0_SERVICE_HOST)/32"
        ],
      "securityContext":
      {
        "privileged": true
      }
    }]'
  labels:
    app: mon
    mon_cluster: rookcluster
    name: mon0
  name: mon0
spec:
  containers:
  - name: socat
    image: quay.io/bzub/socat:latest
    imagePullPolicy: Always
    command:
      - socat
      - -d
      - TCP-LISTEN:$(MON0_SERVICE_PORT_ROOK_MON),bind=$(POD_IP),reuseaddr,fork
      - TCP:$(MON0_SERVICE_HOST):6790,reuseaddr
    env:
    - name: POD_IP
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: status.podIP
  - name: mon
    image: quay.io/rook/rookd:v0.3.1
    command:
    - /bin/sh
    - -c
    - sleep 5;
      /usr/bin/rookd mon
      --private-ipv4=$(MON0_SERVICE_HOST)
      --public-ipv4=$(MON0_SERVICE_HOST)
      --name=mon0
      --mon-endpoints=mon0=$(MON0_SERVICE_HOST):$(MON0_SERVICE_PORT_ROOK_MON),mon1=$(MON1_SERVICE_HOST):$(MON1_SERVICE_PORT_ROOK_MON),mon2=$(MON2_SERVICE_HOST):$(MON2_SERVICE_PORT_ROOK_MON)
      --port=6790
      --fsid=$(ROOK_FSID)
      --cluster-name=$(ROOK_CLUSTER_NAME)
      --ceph-config-override=$(CEPH_CONF)
    env:
    - name: POD_IP
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: status.podIP
    - name: ROOKD_MON_SECRET
      valueFrom:
        secretKeyRef:
          key: mon-secret
          name: mon
    - name: ROOKD_ADMIN_SECRET
      valueFrom:
        secretKeyRef:
          key: admin-secret
          name: mon
    - name: CEPH_CONF
      value: /etc/rook/ceph.conf
    - name: ROOK_FSID
      valueFrom:
        secretKeyRef:
          name: mon
          key: fsid
    - name: ROOK_CLUSTER_NAME
      valueFrom:
        secretKeyRef:
          name: mon
          key: cluster-name
    ports:
    - containerPort: 6790
      name: rook-mon
    volumeMounts:
      - mountPath: /var/lib/rook
        name: rook-data
      - mountPath: /etc/rook
        name: rook-config
        readOnly: true
  nodeName: node1.zbrbdl
  volumes:
    - name: rook-data
      hostPath:
        path: /var/lib/rook-mon
    - name: rook-config
      configMap:
        name: rook-config

rook-mon1-pod.yaml

apiVersion: v1
kind: Pod
metadata:
  annotations:
    rook_version: v0.3.1
    scheduler.alpha.kubernetes.io/affinity: '{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"mon_cluster":"rookcluster"}},"namespaces":null,"topologyKey":"kubernetes.io/hostname"}]}}'
    pod.beta.kubernetes.io/init-containers: '
    [{
      "name": "ip-setup",
      "image": "busybox",
      "command": [
        "sh",
        "-xc", "
          ip addr add dev lo $(MON1_SERVICE_HOST)/32"
        ],
      "securityContext":
      {
        "privileged": true
      }
    }]'
  labels:
    app: mon
    mon_cluster: rookcluster
    name: mon1
  name: mon1
spec:
  containers:
  - name: socat
    image: quay.io/bzub/socat:latest
    imagePullPolicy: Always
    command:
      - socat
      - -d
      - TCP-LISTEN:$(MON1_SERVICE_PORT_ROOK_MON),bind=$(POD_IP),reuseaddr,fork
      - TCP:$(MON1_SERVICE_HOST):6790,reuseaddr
    env:
    - name: POD_IP
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: status.podIP
  - name: mon
    image: quay.io/rook/rookd:v0.3.1
    command:
    - /bin/sh
    - -c
    - sleep 5;
      /usr/bin/rookd mon
      --private-ipv4=$(MON1_SERVICE_HOST)
      --public-ipv4=$(MON1_SERVICE_HOST)
      --name=mon1
      --mon-endpoints=mon0=$(MON0_SERVICE_HOST):$(MON0_SERVICE_PORT_ROOK_MON),mon1=$(MON1_SERVICE_HOST):$(MON1_SERVICE_PORT_ROOK_MON),mon2=$(MON2_SERVICE_HOST):$(MON2_SERVICE_PORT_ROOK_MON)
      --port=6790
      --fsid=$(ROOK_FSID)
      --cluster-name=$(ROOK_CLUSTER_NAME)
      --ceph-config-override=$(CEPH_CONF)
    env:
    - name: POD_IP
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: status.podIP
    - name: ROOKD_MON_SECRET
      valueFrom:
        secretKeyRef:
          key: mon-secret
          name: mon
    - name: ROOKD_ADMIN_SECRET
      valueFrom:
        secretKeyRef:
          key: admin-secret
          name: mon
    - name: CEPH_CONF
      value: /etc/rook/ceph.conf
    - name: ROOK_FSID
      valueFrom:
        secretKeyRef:
          name: mon
          key: fsid
    - name: ROOK_CLUSTER_NAME
      valueFrom:
        secretKeyRef:
          name: mon
          key: cluster-name
    ports:
    - containerPort: 6790
      name: rook-mon
    volumeMounts:
      - mountPath: /var/lib/rook
        name: rook-data
      - mountPath: /etc/rook
        name: rook-config
        readOnly: true
  nodeName: node2.zbrbdl
  volumes:
    - name: rook-data
      hostPath:
        path: /var/lib/rook-mon
    - name: rook-config
      configMap:
        name: rook-config

rook-mon2-pod.yaml

apiVersion: v1
kind: Pod
metadata:
  annotations:
    rook_version: v0.3.1
    scheduler.alpha.kubernetes.io/affinity: '{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"mon_cluster":"rookcluster"}},"namespaces":null,"topologyKey":"kubernetes.io/hostname"}]}}'
    pod.beta.kubernetes.io/init-containers: '
    [{
      "name": "ip-setup",
      "image": "busybox",
      "command": [
        "sh",
        "-xc", "
          ip addr add dev lo $(MON2_SERVICE_HOST)/32"
        ],
      "securityContext":
      {
        "privileged": true
      }
    }]'
  labels:
    app: mon
    mon_cluster: rookcluster
    name: mon2
  name: mon2
spec:
  containers:
  - name: socat
    image: quay.io/bzub/socat:latest
    imagePullPolicy: Always
    command:
      - socat
      - -d
      - TCP-LISTEN:$(MON2_SERVICE_PORT_ROOK_MON),bind=$(POD_IP),reuseaddr,fork
      - TCP:$(MON2_SERVICE_HOST):6790,reuseaddr
    env:
    - name: POD_IP
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: status.podIP
  - name: mon
    image: quay.io/rook/rookd:v0.3.1
    command:
    - /bin/sh
    - -c
    - sleep 5;
      /usr/bin/rookd mon
      --private-ipv4=$(MON2_SERVICE_HOST)
      --public-ipv4=$(MON2_SERVICE_HOST)
      --name=mon2
      --mon-endpoints=mon0=$(MON0_SERVICE_HOST):$(MON0_SERVICE_PORT_ROOK_MON),mon1=$(MON1_SERVICE_HOST):$(MON1_SERVICE_PORT_ROOK_MON),mon2=$(MON2_SERVICE_HOST):$(MON2_SERVICE_PORT_ROOK_MON)
      --port=6790
      --fsid=$(ROOK_FSID)
      --cluster-name=$(ROOK_CLUSTER_NAME)
      --ceph-config-override=$(CEPH_CONF)
    env:
    - name: POD_IP
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: status.podIP
    - name: ROOKD_MON_SECRET
      valueFrom:
        secretKeyRef:
          key: mon-secret
          name: mon
    - name: ROOKD_ADMIN_SECRET
      valueFrom:
        secretKeyRef:
          key: admin-secret
          name: mon
    - name: CEPH_CONF
      value: /etc/rook/ceph.conf
    - name: ROOK_FSID
      valueFrom:
        secretKeyRef:
          name: mon
          key: fsid
    - name: ROOK_CLUSTER_NAME
      valueFrom:
        secretKeyRef:
          name: mon
          key: cluster-name
    ports:
    - containerPort: 6790
      name: rook-mon
    volumeMounts:
      - mountPath: /var/lib/rook
        name: rook-data
      - mountPath: /etc/rook
        name: rook-config
        readOnly: true
  nodeName: node3.zbrbdl
  volumes:
    - name: rook-data
      hostPath:
        path: /var/lib/rook-mon
    - name: rook-config
      configMap:
        name: rook-config