kafka 效能測試

kafka-benchmark.sh

# 安裝JDK
yum install -y java-1.8.0-openjdk java-1.8.0-openjdk-devel

# 安裝 KAFKA
cd /opt
wget http://ftp.tc.edu.tw/pub/Apache/kafka/0.8.2.2/kafka_2.11-0.8.2.2.tgz
tar zxvf kafka_2.11-0.8.2.2.tgz 
cd kafka_2.11-0.8.2.2
# 啟動 ZooKeeper & Kafka
bin/zookeeper-server-start.sh config/zookeeper.properties &
bin/kafka-server-start.sh config/server.properties &


bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic one-broker-benchmark 
# 測試 1k 封包 10w次, 批次100筆
bin/kafka-run-class.sh org.apache.kafka.clients.tools.ProducerPerformance one-broker-benchmark 100000 1024 -1 acks=1 \
bootstrap.servers=localhost:9092 buffer.memory=67108864 batch.size=100
# 測試 1k 封包 10w次, 批次1000筆
bin/kafka-run-class.sh org.apache.kafka.clients.tools.ProducerPerformance one-broker-benchmark 100000 1024 -1 acks=1 \
bootstrap.servers=localhost:9092 buffer.memory=67108864 batch.size=1000
# 測試 1k 封包 10w次, 批次10000筆
bin/kafka-run-class.sh org.apache.kafka.clients.tools.ProducerPerformance one-broker-benchmark 100000 1024 -1 acks=1 \
bootstrap.servers=localhost:9092 buffer.memory=67108864 batch.size=10000

# 測試 1k 封包 50w次, 批次10000筆
bin/kafka-run-class.sh org.apache.kafka.clients.tools.ProducerPerformance one-broker-benchmark  500000 1024 -1 acks=1 \
bootstrap.servers=localhost:9092 buffer.memory=67108864 batch.size=10000
# 測試 1k 封包 50w次, 批次100000筆
bin/kafka-run-class.sh org.apache.kafka.clients.tools.ProducerPerformance one-broker-benchmark 100000 1024 -1 acks=1 \
bootstrap.servers=localhost:9092 buffer.memory=67108864 batch.size=100000

### GCE ###
## default batch.size
# 500000 records sent, 38949.910415 records/sec (38.04 MB/sec), 1419.89 ms avg latency, 2444.00 ms max latency
# , 1269 ms 50th, 2198 ms 95th, 2384 ms 99th, 2438 ms 99.9th.
## batch.size=100000, 1k msg
# 500000 records sent, 67033.114358 records/sec (65.46 MB/sec), 688.68 ms avg latency, 1147.00 ms max latency
# , 844 ms 50th, 1109 ms 95th, 1133 ms 99th, 1145 ms 99.9th.
### MyPC ###
## default batch.size
# 500000 records sent, 37183.014799 records/sec (36.31 MB/sec), 1480.38 ms avg latency, 2278.00 ms max latency
# , 1416 ms 50th, 2081 ms 95th, 2235 ms 99th, 2273 ms 99.9th.
## batch.size=100000, 1k msg
# 500000 records sent, 115767.538782 records/sec (113.05 MB/sec), 367.42 ms avg latency, 641.00 ms max latency
# , 458 ms 50th, 601 ms 95th, 626 ms 99th, 639 ms 99.9th.
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic io-1024-benchmark
bin/kafka-run-class.sh org.apache.kafka.clients.tools.ProducerPerformance io-1024-benchmark 500000 1024 -1 acks=1 \
bootstrap.servers=localhost:9092 buffer.memory=67108864
# 500000 records sent, 38069.133547 records/sec (37.18 MB/sec), 1459.04 ms avg latency, 2466.00 ms max latency, 14
# 02 ms 50th, 2165 ms 95th, 2424 ms 99th, 2461 ms 99.9th.