OSP 11 kafka installer

gistfile1.txt

librdkafka/examples/rdkafka_performance -C -t FAILURE_EVENT_3 -o end -p=3 -b 10.19.110.7:9092,10.19.110.9:9092,10.19.110.11:9092  -u -l   > consumer_final.txt  2>&1
 # producer
 
 librdkafka/examples/rdkafka_performance -P -t FAILURE_EVENT_3 -p=3 -M -b 10.19.110.7:9092,10.19.110.9:9092,10.19.110.11:9092 -u -l -m "PRODUCER SENDING MESSAGE" -s 64 > producer_final.txt  2>&1

 

kafka-installer.sh

#reading
https://kafka.apache.org/documentation/
https://www.confluent.io/blog/how-to-choose-the-number-of-topicspartitions-in-a-kafka-cluster/
https://engineering.linkedin.com/kafka/benchmarking-apache-kafka-2-million-writes-second-three-cheap-machines
https://engineering.linkedin.com/distributed-systems/log-what-every-software-engineer-should-know-about-real-time-datas-unifying
#Get compute and controller nodes
#from jum node
. ~/stackrc

nodes=$(nova list | grep 'compute\|controller' | awk '{print $12}' | cut -d '=' -f 2)

# Install java
for node in $nodes; do  ssh root@$node "yum install java-1.8.0-openjdk-headless -y"; done
#optional install jdk for deibgging purpose
# yum list java*devel*
yum install java-1.8.0-openjdk-devel.x86_64
#for node in $nodes; do  ssh root@$node "java -version"; done

#Install scala
for node in $nodes; do  ssh root@$node "wget http://www.scala-lang.org/files/archive/scala-2.10.3.tgz -O scala-2.10.3.tar.gz"; done

for node in $nodes; do  sshpass -p 'redhat' ssh root@$node "mv scala-2.10.3 /usr/lib"; done

for node in $nodes; do  sshpass -p 'redhat' ssh root@$node "ln -s /usr/lib/scala-2.10.3 /usr/lib/scala"; done


#kafka install
for node in $nodes; do sshpass -p 'redhat'  ssh root@$node "wget http://www.us.apache.org/dist/kafka/0.10.0.1/kafka_2.10-0.10.0.1.tgz -O kafka_2.10-0.10.0.1.tar.gz"; done

for node in $nodes; do  sshpass -p 'redhat' ssh root@$node "tar -xvzf kafka_2.10-0.10.0.1.tar.gz"; done

for node in $nodes; do  sshpass -p 'redhat' ssh root@$node "mv  kafka_2.10-0.10.0.1 /usr/share/kafka "; done

###Configuration
compute_nodes=$(nova list | grep 'compute' | awk '{print $12}' | cut -d '=' -f 2)
for node in $compute_nodes; do sshpass -p 'redhat' ssh root@$node "sed -i 's/^dataDir=.*/dataDir=\/var\/zookeeper\/data/'  /usr/share/kafka/config/zookeeper.properties";done

for node in $compute_nodes; do sshpass -p 'redhat' ssh root@$node "mkdir -p /var/zookeeper/data";done
#IF YOU HAVE JDK INSTALLED, check if its running
for node in $compute_nodes; do sshpass -p 'redhat' ssh root@$node "jps";done
for node in $controller_nodes; do sshpass -p 'redhat' ssh root@$node "jps";done


#zookepper.proeprties

#edit consumer.properties and produc###Nothign to changeer.properts
#producer
#### format: host1:port1,host2:port2 ...
#metadata.broker.list=localhost:9092
#consumer.propertie
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002"
#zookeeper.connect=127.0.0.1:2181

#edit only at controller server.properties
#zookeeper connect
#edit only at zookepper zookepper.propertes
mgmt_compute_nodes=($(grep -A7 -P "NovaCompute" /home/stack/my_templates/ips-from-pool-all.yaml | grep "compute" -A6 | grep  -A5 "external:" | grep -v external | cut -d "-" -f2))

for i,in ${!mgmt_compute_nodes[@]}; do sed -i 's/^server.1=.*/server.$((i+1))${mgmt_compute_nodes[$i]}=2888:3888/'

#get compute nodes
compute_nodes=$(nova list | grep 'compute' | awk '{print $12}' | cut -d '=' -f 2)
#run zoo keeper


###Now run brokers
controller_nodes=$(nova list | grep 'controller' | awk '{print $12}' | cut -d '=' -f 2)


#####Testing Installationpublish
## starting and stopping (START ZOO KEEPER FIRST)
#BROKER FOR CONTROLLER
for node in $controller_nodes; do  sshpass -p 'redhat' ssh root@$node "/usr/share/kafka/bin/kafka-server-stop.sh -daemon /usr/share/kafka/config/server.properties"; done
for node in $controller_nodes; do  sshpass -p 'redhat' ssh root@$node "/usr/share/kafka/bin/kafka-server-start.sh -daemon /usr/share/kafka/config/server.properties"; done
 #ZOO KEPER
for node in $compute_nodes; do  sshpass -p 'redhat' ssh root@$node "/usr/share/kafka/bin/zookeeper-server-stop.sh -daemon /usr/share/kafka/config/zookeeper.properties"; done
for node in $compute_nodes; do  sshpass -p 'redhat' ssh root@$node "/usr/share/kafka/bin/zookeeper-server-start.sh -daemon /usr/share/kafka/config/zookeeper.properties"; done


#publish a message from compute 0
 echo "Hello, Kafka from Redhat" | /usr/share/kafka/bin/kafka-console-producer.sh --producer-property /usr/share/kafka/config/producer.properties --topic New_topic > /dev/null
 
 
#run this in all nodes 
for node in $compute_nodes; do  sshpass -p 'redhat' ssh root@$node "iptables -I INPUT -p tcp -m tcp --dport 3888 -m comment --comment 'zookeeper 3888' -j ACCEPT";done
for node in $compute_nodes; do  sshpass -p 'redhat' ssh root@$node "iptables -I INPUT -p tcp -m tcp --dport 2888 -m comment --comment 'zookeeper 2888' -j ACCEPT";done
for node in $compute_nodes; do  sshpass -p 'redhat' ssh root@$node "iptables -I INPUT -p tcp -m tcp --dport 2181 -m comment --comment 'zookeeper 2181' -j ACCEPT";done
for node in $compute_nodes; do  sshpass -p 'redhat' ssh root@$node "/sbin/service iptables save";done

for node in $control_nodes; do  sshpass -p 'redhat' ssh root@$node "iptables -I INPUT -p tcp -m tcp --dport 3888 -m comment --comment 'zookeeper 3888' -j ACCEPT";done
for node in $control_nodes; do  sshpass -p 'redhat' ssh root@$node "iptables -I INPUT -p tcp -m tcp --dport 2888 -m comment --comment 'zookeeper 2888' -j ACCEPT";done
for node in $control_nodes; do  sshpass -p 'redhat' ssh root@$node "iptables -I INPUT -p tcp -m tcp --dport 2181 -m comment --comment 'zookeeper 2181' -j ACCEPT";done
for node in $control_nodes; do  sshpass -p 'redhat' ssh root@$node "/sbin/service iptables save";done



-A INPUT -p tcp -m tcp --dport 3888 -m comment --comment "zookeeper 3888" -j ACCEPT
-A INPUT -p tcp -m tcp --dport 2888 -m comment --comment "zookeeper 2888" -j ACCEPT
-A INPUT -p tcp -m tcp --dport 2181 -m comment --comment "zookeeper 2181" -j ACCEPT