Kafka benchmark commands

consumer.sh

## Consumer Throughput: Single consumer thread, no compression
## Consumer Throughput: 3 consumer thread, no compression

bin/kafka-consumer-perf-test.sh --topic benchmark-3-3-none \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--messages 15000000 \
--threads 1

producer.sh

# see: https://gist.github.com/jkreps/c7ddb4041ef62a900e6c
# see: https://kafka.apache.org/documentation/#producerconfigs

# Producer Throughput: Single producer thread, no replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 1 \
--partitions 1 \
--topic benchmark-1-1-none

bin/kafka-producer-perf-test.sh --topic benchmark-1-1-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

# Producer Throughput: Single producer thread, no replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 1 \
--partitions 1 \
--topic benchmark-1-1-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-1-1-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196

## Producer Throughput: Single producer thread, 3x asynchronous replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3a-none

bin/kafka-producer-perf-test.sh --topic benchmark-1-3a-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: Single producer thread, 3x asynchronous replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3a-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-1-3a-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196

## Producer Throughput: Single producer thread, 3x synchronous replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3-none

bin/kafka-producer-perf-test.sh --topic benchmark-1-3-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: Single producer thread, 3x synchronous replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-1-3-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196

## Producer Throughput: 3 producer thread, no replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--partitions 1 \
--topic benchmark-3-0-none

bin/kafka-producer-perf-test.sh --topic benchmark-3-0-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: 3 producer thread, no replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--partitions 1 \
--topic benchmark-3-0-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-3-0-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196

## Producer Throughput: 3 producer thread, 3x asynchronous replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3a-none

bin/kafka-producer-perf-test.sh --topic benchmark-3-3a-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: 3 producer thread, 3x asynchronous replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3a-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-3-3a-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196

## Producer Throughput: 3 producer thread, 3x synchronous replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3-none

bin/kafka-producer-perf-test.sh --topic benchmark-3-3-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: 3 producer thread, 3x synchronous replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-3-3-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196