Basic operations with Apache Kafka

kafka-demo.sh

#!/bin/bash
# kafka-demo.sh: A playground for running Kafka on a Docker container
# https://www.tutorialspoint.com/apache_kafka/apache_kafka_basic_operations.htm

# Initialization step
# openjdk:8 image uses the following software:
# Java 1.8.0_212 and Debian 9 (stretch)
docker run --rm -it --name kafka openjdk:8 bash

# All the commands below are run within the Docker container
# To spawn additional shells, run `docker exec -it kafka bash`
# Once kafka directory is configured, run `cd /kafka` before running scripts

# Install useful tools
apt-get update && apt-get install -y less vim

# Setup kafka directory
curl http://archive.apache.org/dist/kafka/0.9.0.0/kafka_2.11-0.9.0.0.tgz -o kafka.tgz
tar -xf kafka.tgz
mv kafka_* kafka

# Start Zookeeper
bin/zookeeper-server-start.sh config/zookeeper.properties # shell

# Start Kafka brokers
bin/kafka-server-start.sh config/server.properties # broker.id=0 port=9092 log.dirs=/tmp/kafka-logs
bin/kafka-server-start.sh config/server-1.properties # broker.id=1 port=9093 log.dirs=/tmp/kafka-logs-1
bin/kafka-server-start.sh config/server-2.properties # broker.id=2 port=9094 log.dirs=/tmp/kafka-logs-2

# Run single-broker demo
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic hello-0
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic hello-0 # shell
bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --zookeeper localhost:2181 --topic hello-0 --from-beginning # shell

# Run multi-broker demo
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic hello-1
bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic hello-1
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic hello-1 # shell
bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --zookeeper localhost:2181 --topic hello-1 --from-beginning # shell

# Alter topic
bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic hello-1 --partitions 2

# Delete topic
bin/kafka-topics.sh --zookeeper localhost:2181 --delete --topic hello-0
bin/kafka-topics.sh --zookeeper localhost:2181 --delete --topic hello-1