Benchmark kafka

kafka_benchmark.sh

#!/bin/bash

# Purpose: To run kafka Kafka Producer / Consumer benchmark for Indix cluster
#

export TOPIC_NAME="benchmark_2_400"
export CLUSTER_SIZE=3
export DRIVE_COUNT=1
export BROKER_ADDRESS="10.181.18.63:9092"
export ZOOKEEPER_ADDRESS="10.181.18.63:2181"
export PARTITION_MULTIPLES=(1 2 3 4 5)
export REPLICATION_MULTIPLES=(1)
export THREAD_MULTIPLES=(1 2 4 8 16 24)
export NUM_RECORDS=(1000 3000 7500 10000 12500 14500 16500 18000 100000 125000 150000)
export RECORD_SIZES=(25600 51200 102400)
export KAFKA_BIN="/apps/kafka/bin/"
export DATE=`date +%y-%m-%d-%H:%M:%S`
export OUTPUT_DIR="/apps/kafka-benchmarks-$DATE"
export KAFKA_HEAP_OPTS="-Xmx10G"

function load_options() {
  read -p "Kafka bin directory (default:$KAFKA_BIN)" TEMP
  if [ ! -z $TEMP ];then
    export KAFKA_BIN=$TEMP;
  fi
  TEMP=""

  read -p "Output directory (default:$OUTPUT_DIR)" TEMP
  if [ ! -z $TEMP ];then
    export OUTPUT_DIR=$TEMP;
  fi
  TEMP=""

  read -p "Broker Address (default:$BROKER_ADDRESS)" TEMP
  if [ ! -z $TEMP ];then
    export BROKER_ADDRESS=$TEMP;
  fi
  TEMP=""

  read -p "Zookeeper Address (default:$ZOOKEEPER_ADDRESS)" TEMP
  if [ ! -z $TEMP ];then
    export ZOOKEEPER_ADDRESS=$TEMP
  fi
  TEMP=""

  read -p "Topic Name Prefix (default:$TOPIC_NAME_PREFIX)" TEMP
  if [ ! -z $TEMP ];then
    export TOPIC_NAME_PREFIX=$TEMP
  fi
  TEMP=""

  read -p "Cluster size (default:$CLUSTER_SIZE)" TEMP
  if [ ! -z $TEMP ];then
    export CLUSTER_SIZE=$TEMP
  fi
  TEMP=""

  read -p "Data drives per node (default:$DRIVE_COUNT)" TEMP
  if [ ! -z $TEMP ];then
    export DRIVE_COUNT=$TEMP
  fi
  TEMP=""
}

#print_intro
#load_options

echo "Broker Address:$BROKER_ADDRESS, Zookeeper Address:$ZOOKEEPER_ADDRESS, Cluster Size:$CLUSTER_SIZE, Drive Count:$DRIVE_COUNT"

mkdir -p $OUTPUT_DIR
echo "bootstrap.servers=$BROKER_ADDRESS" > $OUTPUT_DIR/producer.conf


echo -e "Running Benchmarks: Topic: $TOPIC_NAME, Partitions:$PARTITION_COUNT, Replication: $k\nResults will be stored in $OUTPUT_DIR/bench-output.txt"
mkdir -p "$OUTPUT_DIR/$TOPIC_NAME"

for i in ${NUM_RECORDS[@]}
  do
    echo "Now running for num-records = $i"
    for rs in ${RECORD_SIZES[@]}
      do
       echo "Now running for record size = $rs"
       $KAFKA_BIN/kafka-producer-perf-test.sh --topic $TOPIC_NAME --num-records $i --record-size $rs --producer-props bootstrap.servers=10.181.18.171:9092,10.181.18.63:9092,10.181.18.57:9092 buffer.memory=671088640 compression.type=lz4 batch.size=20971520 retries=3 max.request.size=20971520 linger.ms=50 --throughput 15000000 --print-metrics  >> "$OUTPUT_DIR/bench-output.txt"
      done
done