SetupKafkaZookeeper-RPM.sh

#!/bin/bash
# Author : Mansur Ul Hasan 
# Email : [email protected] 
# LinkedIn : https://www.linkedin.com/in/mansurulhasan/

### This script setup kafka and zookeeper and well tested on CentOS Amazon Linux and FEDORA
# USAGE of the script can be standalone and could be use for cluster as well
# To Install Zookeeper on the server download the script and run with root user
# sh SetupKafkaZookeeper-RPM.sh  --setup-zookeeper
# follow the instructions and complete the zookeeper setup 
# 
# To Install Kafka on the server download the script and run with root user
# sh SetupKafkaZookeeper-RPM.sh --setup-kafka
# follow the instructions and complete the zookeeper setup 

function DownloadPrereq() {
# This function download the binaries 
	echo "Downloading kafka_2.13-2.8.0 prerequsites ..."
	mkdir -p /tmp/kafka && cd /tmp/kafka
	wget https://downloads.apache.org/kafka/2.8.0/kafka_2.13-2.8.0.tgz
	wget https://gist.githubusercontent.com/mansurali901/47611d3b0ada63d39ba89b1cc8275d12/raw/d5e0e025a8ee9803fb9b1f22b6c33f572ec395b6/setup_jdk1.8.sh
}

function SetupBinaries() {
# This function excute downloaded binaries and install them on server
	DownloadPrereq
	echo 2
	sh /tmp/kafka/setup_jdk1.8.sh
	sleep 3
	mkdir /opt/kafka-latest
	tar -xvf /tmp/kafka/kafka_2.13-2.8.0.tgz -C /opt/kafka-latest
	adduser --system kafka --shell=/sbin/nologin
	cd /opt/kafka-latest && mv  kafka_2.13-2.8.0/* . && rm -rvf kafka_2.13-2.8.0
	chown -R kafka:kafka /opt/kafka-latest

}

function SetupZookeeper() {
# This portions is specifically for Zookeeper Setup 
	SetupBinaries
	echo "Adding Zookeeper properties "
	read -p "Enter zookeeper port " zkport
	echo "# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# the directory where the snapshot is stored.
dataDir=/tmp/zookeeper
# the port at which the clients will connect
clientPort=<ZOOKEEPERPORT>
# disable the per-ip limit on the number of connections since this is a non-production config
maxClientCnxns=0
# Disable the adminserver by default to avoid port conflicts.
# Set the port to something non-conflicting if choosing to enable this
admin.enableServer=false
# admin.serverPort=8080" > /opt/kafka-latest/config/zookeeper.properties

	sed -i "s@<ZOOKEEPERPORT>@$zkport@g" /opt/kafka-latest/config/zookeeper.properties
	echo "Creating Service ..."
	echo "
[Unit]
Description=Apache Zookeeper server (Kafka)
Documentation=http://zookeeper.apache.org
Requires=network.target remote-fs.target
After=network.target remote-fs.target

[Service]
Type=simple
User=kafka
Group=kafka
ExecStart=/opt/kafka-latest/bin/zookeeper-server-start.sh /opt/kafka-latest/config/zookeeper.properties
ExecStop=/opt/kafka-latest/bin/zookeeper-server-stop.sh

[Install]
WantedBy=multi-user.target" > /lib/systemd/system/zookeeper.service
	chmod 644 /lib/systemd/system/zookeeper.service
	systemctl daemon-reload && systemctl enable zookeeper && systemctl start zookeeper
}

function SetupKafka() {
# This Portion is specifically for Kafka Setup 
	SetupBinaries
	echo "Adding Kafka properties ..."
	#read -p "Add number of Brokers : " nb
	read -p " Enter Zookeeper Address : : " zkhostname
	#for (( i = 1; i <= $nb; ++i ));
	#do
	        echo "Starting Kafka Setup "
		read -p "Enter Internal Hostname for Broker  : " ihname
		read -p "Enter External Hostname for Broker  : " ehname
		read -p "Enter Broker ID : " id

		echo "
# The id of the broker. This must be set to a unique integer for each broker.
broker.id=<id>
listeners=LISTENER_INTERNAL://<INTERNAL_HOSTNAME>:29092,LISTENER_EXTERNAL://<EXTERNAL_HOSTNAME>:9092
advertised.listeners=LISTENER_INTERNAL://<INTERNAL_HOSTNAME>:29092,LISTENER_EXTERNAL://<EXTERNAL_HOSTNAME>:9092
listener.security.protocol.map=LISTENER_INTERNAL:PLAINTEXT,LISTENER_EXTERNAL:PLAINTEXT
inter.broker.listener.name=LISTENER_INTERNAL

# The number of threads that the server uses for receiving requests from the network and sending responses to the network
num.network.threads=3

# The number of threads that the server uses for processing requests, which may include disk I/O
num.io.threads=8

# The send buffer (SO_SNDBUF) used by the socket server
socket.send.buffer.bytes=102400

# The receive buffer (SO_RCVBUF) used by the socket server
socket.receive.buffer.bytes=102400

# The maximum size of a request that the socket server will accept (protection against OOM)
socket.request.max.bytes=104857600

# A comma separated list of directories under which to store log files
log.dirs=/tmp/kafka-logs
# default value of partitions
num.partitions=1
num.recovery.threads.per.data.dir=1

offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1

# The minimum age of a log file to be eligible for deletion due to age
log.retention.hours=168

# The maximum size of a log segment file. When this size is reached a new log segment will be created.
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000

zookeeper.connect=<ZOOKEEPER_HOST>:2181
# Timeout in ms for connecting to zookeeper
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0" > /opt/kafka-latest/config/server.properties

	echo "Editing Kafka Server properties .. "
	sed -i "s@<INTERNAL_HOSTNAME>@$ihname@g" /opt/kafka-latest/config/server.properties
	sed -i "s@<EXTERNAL_HOSTNAME>@$ehname@g" /opt/kafka-latest/config/server.properties
	sed -i "s@<ZOOKEEPER_HOST>@$zkhostname@g" /opt/kafka-latest/config/server.properties
	sed -i "s@<id>@$id@g" /opt/kafka-latest/config/server.properties
	echo "Creating Service File ..."
	echo "
Description=Apache Kafka server (Kafka)
Documentation=http://zookeeper.apache.org
Requires=network.target remote-fs.target
After=network.target remote-fs.target

[Service]
Type=simple
User=kafka
Group=kafka
ExecStart=/opt/kafka-latest/bin/kafka-server-start.sh /opt/kafka-latest/config/server.properties
ExecStop=/opt/kafka-latest/bin/kafka-server-stop.sh

[Install]
WantedBy=multi-user.target
" > /usr/lib/systemd/system/kafka.service
	chmod 644 /usr/lib/systemd/system/kafka.service
	systemctl daemon-reload && systemctl enable kafka && systemctl start kafka
}

case $1 in
"--setup-kafka")
	SetupKafka
;;
"--setup-zookeeper")
	SetupZookeeper
;;
"ping")
	echo "pong"
	;;
"--help")
echo "
Usage of available options
--setup-kafka		To Setup Kafka on Linux Servers
--setup-zookeeper	To Setup Zookeeper on Linux Servers
"
;;
*)
echo "
Usage of available options
--setup-kafka           To Setup Kafka on Linux Servers
--setup-zookeeper       To Setup Zookeeper on Linux Servers
"
;;
esac