Kafka aws Scripts

kafka_aws_setup.sh

# Java 8 setup
sudo apt-get update
sudo apt-get -y install default-jdk

# install kafka
wget https://apache.mirror.digionline.de/kafka/2.7.0/kafka_2.12-2.7.0.tgz
tar -xzf kafka_2.12-2.7.0.tgz
rm -rf kafka_2.12-2.7.0.tgz

# start zookeeper
cd kafka_2.12-2.7.0
bin/zookeeper-server-start.sh -daemon config/zookeeper.properties

# edit public listener
nano config/server.properties

advertised.listeners=PLAINTEXT://<kafkahostname>:9092
# advertised.listeners=PLAINTEXT://ec2-18-184-134-47.eu-central-1.compute.amazonaws.com:9092 

# start kafka
cd kafka_2.12-2.7.0
bin/kafka-server-start.sh -daemon config/server.properties

# create topic
bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic test
# or
bin/kafka-topics.sh --create --topic quickstart-events --bootstrap-server localhost:9092

# Test topic
bin/kafka-topics.sh --describe --topic test --bootstrap-server localhost:9092

# produce intern
bin/kafka-console-producer.sh --topic test --bootstrap-server localhost:9092
# produce extern
bin/kafka-console-producer.sh --topic test --bootstrap-server ec2-18-184-134-47.eu-central-1.compute.amazonaws.com:9092

# consume 
bin/kafka-console-consumer.sh --topic test --from-beginning --bootstrap-server localhost:9092


# stop
bin/kafka-server-stop.sh
bin/zookeeper-server-stop.sh