Zero downtime on statefulset update

README.md

1. Create cluster

$ gcloud container clusters create aojea

2. Install the statefulset and wait until it is ready

$ kubectl apply -f test.yaml
statefulset.apps/apache created
service/apache created
$ kubectl rollout status statefulset apache
partitioned roll out complete: 2 new pods have been updated...

3. Test connectivity

# Within the Cluster and using IPs ( avoid DNS by now)
$  kubectl get service apache
NAME     TYPE           CLUSTER-IP    EXTERNAL-IP   PORT(S)        AGE
apache   LoadBalancer   10.100.2.65   10.154.0.14   80:31350/TCP   6m46s
$ kubectl run -it test-pod --image registry.k8s.io/e2e-test-images/agnhost:2.39 --command -- bash
If you don't see a command prompt, try pressing enter.
bash-5.0# while true; do curl -sS http://10.154.0.14/hostname; echo; sleep 1; done
# In parallel restart the statefulset
$ kubectl rollout restart statefulset apache
# Wait until rollout finish and check there are no errors on the test pod
$ kubectl rollout status statefulset apache
Waiting for 1 pods to be ready...
Waiting for 1 pods to be ready...
partitioned roll out complete: 2 new pods have been updated...
# The upgrade has to be ordered, pods are restarted sequentually in a statefulset
# check the test-pod output
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-1
apache-0
apache-0
apache-0
apache-0
apache-0
apache-1
apache-1
apache-1
apache-0
apache-1
apache-1

4. Test from outside the cluster

# Create a vm
gcloud compute instances create testvm \
    --image-family=debian-11 \
    --image-project=debian-cloud \
    --machine-type=e2-small
gcloud compute firewall-rules create allow-ssh --direction=INGRESS --priority=1000 --network=default --action=ALLOW --rules=tcp:22 --source-ranges=0.0.0.0/0
gcloud compute ssh testvm
while true; do curl -sS http://10.154.0.14/hostname; echo; sleep 1; done
aojea@testvm:~$ while true; do curl  --connect-timeout 5 http://10.128.0.21/hostname; echo; sleep 1; done
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-0
apache-1
apache-0
apache-0
apache-1
apache-0
apache-0
apache-0
apache-0
apache-1
apache-1
apache-0
apache-1
apache-0
apache-1
apache-1
apache-1
apache-1
apache-0
apache-1
apache-1
apache-1
apache-0
apache-0
apache-1
apache-0
apache-1
apache-1

test.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: apache
spec:
  selector:
    matchLabels:
      app: apache
  serviceName: apache
  replicas: 2
  template:
    metadata:
      labels:
        app: apache
    spec:
      terminationGracePeriodSeconds: 120
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - nginx
            topologyKey: kubernetes.io/hostname
      containers:
      - name: apache
        image: registry.k8s.io/e2e-test-images/agnhost:2.39
        args:
          - netexec
          - --http-port=80
          - --delay-shutdown=45
        ports:
          - name: httpd
            containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: apache
  annotations:
    networking.gke.io/load-balancer-type: Internal
spec:
  type: LoadBalancer
  externalTrafficPolicy: Local
  selector:
    app: apache
  ports:
    - name: httpd
      port: 80
      targetPort: 80