KAFKA & ZOOKEEPER cluster

KAFKA & ZOOKEEPER cluster

1. 10.204.11.181
2. 10.204.11.182
3. 10.204.11.183

wget https://downloads.apache.org/kafka/3.6.1/kafka_2.13-3.6.1.tgz

wget https://dlcdn.apache.org/zookeeper/zookeeper-3.9.1/apache-zookeeper-3.9.1-bin.tar.gz

tar xvzf kafka_2.13-3.6.1.tgz

tar xvzf apache-zookeeper-3.9.1-bin.tar.gz

mv kafka_2.13-3.6.1.tgz /opt/kafka
mv apache-zookeeper-3.9.1-bin.tar.gz /opt/zookeeper
====================================================web-ui=================================================
apt install docker.io

docker run -d -it -p 8085:8080 -e DYNAMIC_CONFIG_ENABLED=true provectuslabs/kafka-ui:master
==============================================KAFKA SYSTEMD================================================
[Unit]
Requires=kafka.service
After=kafka.service

[Service]
Type=simple
User=root
ExecStart=/bin/sh -c '/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties'
ExecStop=/opt/kafka/bin/kafka-server-stop.sh
Restart=on-abnormal

[Install]
WantedBy=multi-user.target
===============================================ZOOKEEPER SYSTEMD===========================================
[Unit]
Description=Zookeeper Daemon
Documentation=http://zookeeper.apache.org
Requires=network.target
After=network.target

[Service]
Type=forking
WorkingDirectory=/opt/zookeeper
User=root
Group=root
ExecStart=/opt/zookeeper/bin/zkServer.sh start /opt/zookeeper/conf/zoo.cfg
ExecStop=/opt/zookeeper/bin/zkServer.sh stop /opt/zookeeper/conf/zoo.cfg
ExecReload=/opt/zookeeper/bin/zkServer.sh restart /opt/zookeeper/conf/zoo.cfg
TimeoutSec=30
Restart=on-failure

[Install]
WantedBy=default.target

=============================================srv-01 /opt/kafka/config/server.properties==========================================
broker.id=1
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
# The maximum size of a request that the socket server will accept (protection against OOM)
#socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
listeners=PLAINTEXT://:9092
advertised.host.name=10.204.11.181
advertised.port=9092
advertised.listeners=PLAINTEXT://10.204.11.181:9092
zookeeper.connect=10.204.11.181:2181,10.204.11.182:2181,10.204.11.183:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
=============================================srv-02 /opt/kafka/config/server.properties==========================================
broker.id=2
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
# The maximum size of a request that the socket server will accept (protection against OOM)
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=30000
listeners=PLAINTEXT://:9092
advertised.host.name=10.204.11.182
advertised.port=9092
advertised.listeners=PLAINTEXT://10.204.11.182:9092
zookeeper.connect=10.204.11.181:2181,10.204.11.182:2181,10.204.11.183:2181
zookeeper.connection.timeout.ms=18000
=============================================srv-03 /opt/kafka/config/server.properties==========================================
broker.id=3
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
# The maximum size of a request that the socket server will accept (protection against OOM)
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
advertised.host.name=10.204.11.183
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://10.204.11.183:9092
zookeeper.connect=10.204.11.181:2181,10.204.11.182:2181,10.204.11.183:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
======================================srv-01 /opt/zookeeper/conf/zoo.cfg==========================================================
tickTime=2000
dataDir=/var/lib/zookeeper
clientPort=2181
initLimit=5
syncLimit=2
server.1=10.204.11.181:2888:3888
server.2=10.204.11.182:2888:3888
server.3=10.204.11.183:2888:3888
======================================srv-02 /opt/zookeeper/conf/zoo.cfg==========================================================
tickTime=2000
dataDir=/var/lib/zookeeper
clientPort=2181
initLimit=5
syncLimit=2
server.1=10.204.11.181:2888:3888
server.2=10.204.11.182:2888:3888
server.3=10.204.11.183:2888:3888
======================================srv-03 /opt/zookeeper/conf/zoo.cfg==========================================================
tickTime=2000
dataDir=/var/lib/zookeeper
clientPort=2181
initLimit=5
syncLimit=2
server.1=10.204.11.181:2888:3888
server.2=10.204.11.182:2888:3888
server.3=10.204.11.183:2888:3888
==========================================================zookeeper node id=============================================================

mkdir -p /var/lib/zookeeper/
touch /var/lib/zookeeper/myid

echo "1" > /var/lib/zookeeper/myid
echo "2" > /var/lib/zookeeper/myid
echo "3" > /var/lib/zookeeper/myid
========================================================================================================================================

systemctl start zookeeper
systemctl start kafka


systemctl status kafka
systemctl status zookeeper

systemctl enable kafka
systemctl enable zookeeper