Kafka Performance Testing Scripts (Producer / Batching)

grid_scan_batching.sh

#!/usr/bin/env bash
extract_latency() {
    echo $1 | tr ',' '\n' | grep 'avg latency' | sed 's/ ms avg latency//g'
}

extract_throughput() {
  echo $1 | tr ',' '\n' | grep 'records/sec' | cut -d "(" -f2 | cut -d ")" -f1 | sed 's/ MB\/sec//g'
}

batch_size_arr=(0 10 100 1000 10000 100000 1000000)
linger_ms_arr=(0 10 30 100 300 1000 3000)

for batch_size in ${batch_size_arr[@]}; do
  for linger_ms in ${linger_ms_arr[@]}; do
      perf_out=$(kafka-producer-perf-test --topic performance --num-records 1000000 --record-size 100 --throughput 10000000 --producer-props bootstrap.servers=kafka-1:9092,kafka-2:9092,kafka-3:9092 acks=all linger.ms=$linger_ms batch.size=$batch_size)
      echo $batch_size $linger_ms $(extract_latency($perf_out)) $(extract_throughput($perf_out))
  done
done

linger_scan.sh

#!/usr/bin/env bash

size_arr=(0 10 100 1000 10000 100000 1000000)

for size in ${size_arr[@]}; do
    echo $size
    kafka-producer-perf-test --topic performance --num-records 1000000 --record-size 100 --throughput 10000000 --producer-props bootstrap.servers=kafka-1:9092,kafka-2:9092,kafka-3:9092 acks=all batch.size=$size linger.ms=500
done