Benchmark Storage Class

benchmark-job.k8s.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: io-benchmark
  labels:
    app.kubernetes.io/name: io-benchmark
data:
  rand-rw.fio: |
    [global]
    bs=4K
    iodepth=256
    direct=1
    ioengine=libaio
    group_reporting
    time_based
    runtime=120
    numjobs=4
    directory=/iops/testvol

    [job0]
    name=rand-rw
    size=1g
    rw=randrw
  rand-read.fio: |
    [global]
    bs=4K
    iodepth=256
    direct=1
    ioengine=libaio
    group_reporting
    time_based
    runtime=120
    numjobs=4
    directory=/iops/testvol

    [job0]
    name=rand-read
    size=1g
    rw=randread
  rand-write.fio: |
    [global]
    bs=4K
    iodepth=256
    direct=1
    ioengine=libaio
    group_reporting
    time_based
    runtime=120
    numjobs=4
    directory=/iops/testvol

    [job0]
    name=rand-write
    size=1g
    rw=randwrite
  seq-rw.fio: |
    [global]
    bs=4K
    iodepth=256
    direct=1
    ioengine=libaio
    group_reporting
    time_based
    runtime=120
    numjobs=4
    directory=/iops/testvol

    [job0]
    name=seq-rw
    size=1g
    rw=rw
  seq-read.fio: |
    [global]
    bs=4K
    iodepth=256
    direct=1
    ioengine=libaio
    group_reporting
    time_based
    runtime=120
    numjobs=4
    directory=/iops/testvol

    [job0]
    name=seq-read
    size=1g
    rw=read
  seq-write.fio: |
    [global]
    bs=4K
    iodepth=256
    direct=1
    ioengine=libaio
    group_reporting
    time_based
    runtime=120
    numjobs=4
    directory=/iops/testvol

    [job0]
    name=seq-write
    size=1g
    rw=write
---
apiVersion: batch/v1
kind: Job
metadata:
  name: io-benchmark
  labels:
    app.kubernetes.io/name: io-benchmark
spec:
  template:
    metadata:
      labels:
        app.kubernetes.io/name: io-benchmark
    spec:
      securityContext:
        fsGroup: 1000
      initContainers:
        - name: io-bench-latency
          image: ghcr.io/comet-ml/docker-iops:1.0.0
          command:
            - /bin/sh
          args:
            - -ec
            - |
              docker-entrypoint.sh ioping -J -c 16 /iops/testvol | tee /iops/output/latency.json
          volumeMounts:
            - mountPath: /iops/testvol
              name: testvol
            - mountPath: /iops/output
              name: output
        - name: io-bench-randrw
          image: ghcr.io/comet-ml/docker-iops:1.0.0
          args:
            - fio
            - /iops/cfg/rand-rw.fio
            - --output=/iops/output/rand-rw.json
            - --output-format=json
          volumeMounts:
            - mountPath: /iops/cfg
              name: jobcfg
            - mountPath: /iops/testvol
              name: testvol
            - mountPath: /iops/output
              name: output
        - name: io-bench-randread
          image: ghcr.io/comet-ml/docker-iops:1.0.0
          args:
            - fio
            - /iops/cfg/rand-read.fio
            - --output=/iops/output/rand-read.json
            - --output-format=json
          volumeMounts:
            - mountPath: /iops/cfg
              name: jobcfg
            - mountPath: /iops/testvol
              name: testvol
            - mountPath: /iops/output
              name: output
        - name: io-bench-randwrite
          image: ghcr.io/comet-ml/docker-iops:1.0.0
          args:
            - fio
            - /iops/cfg/rand-write.fio
            - --output=/iops/output/rand-write.json
            - --output-format=json
          volumeMounts:
            - mountPath: /iops/cfg
              name: jobcfg
            - mountPath: /iops/testvol
              name: testvol
            - mountPath: /iops/output
              name: output
        - name: io-bench-seqrw
          image: ghcr.io/comet-ml/docker-iops:1.0.0
          args:
            - fio
            - /iops/cfg/seq-rw.fio
            - --output=/iops/output/seq-rw.json
            - --output-format=json
          volumeMounts:
            - mountPath: /iops/cfg
              name: jobcfg
            - mountPath: /iops/testvol
              name: testvol
            - mountPath: /iops/output
              name: output
        - name: io-bench-seqread
          image: ghcr.io/comet-ml/docker-iops:1.0.0
          args:
            - fio
            - /iops/cfg/seq-read.fio
            - --output=/iops/output/seq-read.json
            - --output-format=json
          volumeMounts:
            - mountPath: /iops/cfg
              name: jobcfg
            - mountPath: /iops/testvol
              name: testvol
            - mountPath: /iops/output
              name: output
        - name: io-bench-seqwrite
          image: ghcr.io/comet-ml/docker-iops:1.0.0
          args:
            - fio
            - /iops/cfg/seq-write.fio
            - --output=/iops/output/seq-write.json
            - --output-format=json
          volumeMounts:
            - mountPath: /iops/cfg
              name: jobcfg
            - mountPath: /iops/testvol
              name: testvol
            - mountPath: /iops/output
              name: output
      containers:
        - name: report
          image: leplusorg/json
          command:
            - sh
          args:
            - -ec
            - |
              jq -s . /iops/output/*.json
          volumeMounts:
            - mountPath: /iops/output
              name: output
      restartPolicy: Never
      volumes:
        - name: jobcfg
          configMap:
            name: io-benchmark
        - name: testvol
          ephemeral:
            volumeClaimTemplate:
              spec:
                accessModes:
                  - ReadWriteOnce
                resources:
                  requests:
                    storage: 32Gi
                storageClassName: standard
                volumeMode: Filesystem
        - name: output
          emptyDir: {}
  backoffLimit: 0

benchmark.sh

#!/bin/bash

set -e

if [ -z "$1" ]
then
	echo "Usage:"
	echo -e "\t${0} <storage class name>"
	exit 2
fi

if [[ "$1" =~ [[:space:]] ]]
then
	echo "Given Storage Class Name is not valid!"
	echo -e "\t'$1'"
	exit 1
fi

yq -i "select(di == 1).spec.template.spec.volumes[1].ephemeral.volumeClaimTemplate.spec.storageClassName = \"${1}\"" benchmark-job.k8s.yaml

kubectl apply -f benchmark-job.k8s.yaml

sleep 1

kubectl get pod -l app.kubernetes.io/name=io-benchmark -w &

cleanup() {
	kill %1
}
trap cleanup SIGINT

kubectl wait --for=condition=complete --timeout=15m job/io-benchmark

kill %1
trap - SIGINT

kubectl logs -l app.kubernetes.io/name=io-benchmark -c report --tail=-1 > bench-${1}.json

kubectl delete -f benchmark-job.k8s.yaml

less bench-${1}.json