Test Kubernetes pod deletion time in a loop

pod-delete-test.sh

#!/bin/bash

# Copyright 2018 Robert Krawitz
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# Run loop of pod create/delete.

#export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig
KUBECTL=kubectl

declare -i delay=0
declare -i count=1
declare -i pod_delay=0
declare -i pause=0
declare    base_podname=failpod
declare -i pod_status=2
declare    grace_period
declare -i iteration=0
declare -i iterations_complete=0
declare -i parallel=1
declare    force=
declare    unique=
declare    podname=
declare    verbose=
declare -i min_time=-1
declare -i max_time=0
declare -i tot_timer=0
declare -i openshift=0

function help() {
    cat <<EOF
Usage: loop-pod options...
  Options:
    -t N=$delay      Delay in seconds between starting and deleting pod
    -n N=$count      Number of cycles to run
    -d N=$pod_delay      Delay in seconds before pod exits
    -p N=$pause      Pause between cycles
    -N name     Name of pod ($podname)
    -x N=$pod_status      Exit status of container
    -g N        Termination grace period (no default)
    -f          Force delete pods
    -P N=$parallel      Run multiple jobs concurrently
    -u          Name each pod uniquely
    -K          Use Kubernetes to create pod (default)
    -O          Use OpenShift rather than Kubernetes
    -v          Print verbose status
EOF
    exit 1
}

while getopts "t:n:d:p:N:x:g:P:uvOfh" opt ; do
    case "$opt" in
	t) delay="$OPTARG"        ;;
	n) count="$OPTARG"        ;;
	d) pod_delay="$OPTARG"    ;;
	p) pause="$OPTARG"        ;;
	N) base_podname="$OPTARG" ;;
	x) pod_status="$OPTARG"   ;;
	g) grace_period="$OPTARG" ;;
	f) force='--force'        ;;
	P) parallel="$OPTARG"     ;;
	K) openshift=0            ;;
	O) openshift=1            ;;
	u) unique=1               ;;
	v) verbose=1              ;;
	h) help                   ;;
	*) help                   ;;
    esac
done

(( $openshift > 0 )) && KUBECTL=oc

declare -i iteration=0

function generate_podname() {
    echo "${base_podname}${unique:+-$iteration}"
}

function dstamp() {
    date +%s
}

function generate_pod() {
    cat <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: "$podname"
spec:
  containers:
  - name: busybox
    image: busybox
    command:
    - /bin/sh
    - "-c"
    - "sleep $pod_delay; exit $pod_status"
EOF
}

(($count == 0)) && count=1000000

if ((parallel > 1)) ; then
    pstuff=" * $parallel jobs"
    unique=1
    verbose=
fi

echo "Running $count passes$pstuff, delay $delay, pod exit delay $pod_delay, pause between runs $pause${grace_period:+ grace $grace_period}"

grace_period=${grace_period:+--grace-period=$grace_period}

function finis() {
    (($iteration > 0)) && echo "$iterations_complete passes, min $min_time, max $max_time, average $(( ($tot_timer + ($iterations_complete / 2)) / $iterations_complete ))"
    exit
}

function run_iteration() {
    local iteration=$1
    [[ $iteration > 1 ]] && sleep $pause
    echo -n "Pass $iteration: "
    podname=$(generate_podname)
    $KUBECTL create -f - <<< "$(generate_pod)" >/dev/null
    if [[ -n $verbose ]] ; then
	declare firsttime=1
	declare -i start=$(dstamp)
	declare -i end=$((start + delay))
	while (( $(dstamp) < $end )) ; do
	    if [[ -n $firsttime ]] ; then
		echo
		$KUBECTL get pod "$podname"
		firsttime=
	    else
		$KUBECTL get pod "$podname" --no-headers
	    fi		
	    sleep 1
	done
    else
	sleep $delay
	echo -n "|$($KUBECTL get pod "$podname" --no-headers)|"
    fi
    start=$(dstamp)
    if [[ -n $verbose ]] ; then
	$KUBECTL delete pod "$podname" $grace_period $force
	while $KUBECTL get pod "$podname" --no-headers 1>&2 ; do
	    sleep 1
	done
    else
	$KUBECTL delete pod "$podname" $grace_period $force >/dev/null 2>&1
	while $KUBECTL get pod "$podname" --no-headers >/dev/null 2>&1 ; do
	    sleep 1
	done
    fi
    echo " $(($(dstamp) - start)) seconds"
}

function run_job() {
    trap finis EXIT HUP INT TERM
    local worker=$1
    for iteration in $(seq 0 $((count-1))) ; do
	output=$(run_iteration $((worker+(parallel*iteration))))
	declare -a ostuff=($output)
	timer=${ostuff[7]}
	if [[ -n $timer ]] ; then
	    (( $min_time < 0 || $timer < $min_time )) && min_time=$(($timer))
	    (( $timer > $max_time )) && max_time=$timer
	    tot_timer+=$timer
	    iterations_complete+=1
	fi
	echo "$output"
    done
    finis
}

for worker in $(seq 1 $parallel) ; do
    run_job $worker &
done
wait

Example:

pod-delete-test.sh -n 1000 -t 7 -u -N apod

runs 1000 iterations of kubectl create/kubectl delete, with a delay of 7 seconds between creation and deletion of the pods (to give the pod time to run), with unique names for each pod (to simplify inspection of the log), and a prefix of "apod" for the pod names.

It's possible to run multiple copies of this concurrently with different parameters; in this case, you should use different -N options for each copy.