deploy.sh

#!/usr/bin/env bash

sudo yum install -y https://centos7.iuscommunity.org/ius-release.rpm
sudo yum update -y
sudo yum install xz-devel -y
sudo yum install -y python34u python34u-libs python34u-devel python34u-pip

#sudo yum install centos-release-scl -y
#sudo yum install rh-python34 -y
# need to run outside .    scl enable rh-python34 bash
sudo yum groupinstall 'Development Tools' -y

cd /root

# Install and setup EFM
# We need it after cluster start as we want NiFi Registry to be available
wget -q https://archive.cloudera.com/CEM/centos7/1.x/updates/1.0.0.0/CEM-1.0.0.0-centos7-tars-tarball.tar.gz -O /root/CEM-1.0.0.0-centos7-tars-tarball.tar.gz
tar -xzf /root/CEM-1.0.0.0-centos7-tars-tarball.tar.gz
mkdir /etc/efm
tar -xzf /root/CEM/centos7/1.0.0.0-54/tars/efm/efm-1.0.0.1.0.0.0-54-bin.tar.gz -C /etc/efm
chown -R root:root /etc/efm
export EFM_HOME='/etc/efm/efm-1.0.0.1.0.0.0-54/'
export PRIVATE_IP=$(hostname --ip-address)
sed -i "[email protected][email protected]=0.0.0.0@g" ${EFM_HOME}/conf/efm.properties
sed -i "[email protected][email protected]=true@g" ${EFM_HOME}/conf/efm.properties
sed -i "[email protected]=http://localhost:[email protected]=http://demo.hortonworks.com:61080@g" ${EFM_HOME}/conf/efm.properties
sed -i "[email protected][email protected]=efm@g" ${EFM_HOME}/conf/efm.properties

# Install and setup MiNiFi-cpp
mkdir /etc/minifi-cpp
tar -xzf /root/CEM/centos7/1.0.0.0-54/tars/nifi-minifi-cpp/nifi-minifi-cpp-0.6.0-bin.tar.gz -C /etc/minifi-cpp/
chown -R root:root /etc/minifi-cpp
export MINIFICPP_HOME='/etc/minifi-cpp/nifi-minifi-cpp-0.6.0'
sed -i "s@#[email protected].@g" ${MINIFICPP_HOME}/conf/minifi.properties
sed -i "[email protected][email protected]=RESTSender@g" ${MINIFICPP_HOME}/conf/minifi.properties
sed -i "[email protected][email protected]=minificpp@g" ${MINIFICPP_HOME}/conf/minifi.properties
sed -i "[email protected][email protected]=http://demo.hortonworks.com:10080/efm/api/c2-protocol/heartbeat@g" ${MINIFICPP_HOME}/conf/minifi.properties
sed -i "[email protected][email protected]=http://demo.hortonworks.com:10080/efm/api/c2-protocol/acknowledge@g" ${MINIFICPP_HOME}/conf/minifi.properties


# Install and setup MiNiFi-java
mkdir /etc/minifi-java
tar -xzf /root/CEM/centos7/1.0.0.0-54/tars/minifi/minifi-0.6.0.1.0.0.0-54-bin.tar.gz -C /etc/minifi-java/
chown -R root:root /etc/minifi-java
export MINIFIJAVA_HOME='/etc/minifi-java/minifi-0.6.0.1.0.0.0-54'
wget -q https://s3.eu-west-2.amazonaws.com/whoville/v2/nifi-kafka-2-0-nar-1.8.0.nar -O ${MINIFIJAVA_HOME}/lib/nifi-kafka-2-0-nar-1.8.0.nar
wget -q https://s3.eu-west-2.amazonaws.com/whoville/v2/nifi-standard-services-api-nar-1.8.0.nar ${MINIFIJAVA_HOME}/lib/nifi-standard-services-api-nar-1.8.0.nar
sed -i "s@#[email protected].@g" ${MINIFIJAVA_HOME}/conf/bootstrap.conf
sed -i "[email protected][email protected]=minifijava@g" ${MINIFIJAVA_HOME}/conf/bootstrap.conf
sed -i "[email protected][email protected]=http://demo.hortonworks.com:10080/efm/api/c2-protocol/heartbeat@g" ${MINIFIJAVA_HOME}/conf/bootstrap.conf
sed -i "[email protected][email protected]=http://demo.hortonworks.com:10080/efm/api/c2-protocol/acknowledge@g" ${MINIFIJAVA_HOME}/conf/bootstrap.conf


# Handle NiFi flow etc. setup
export PYTHONHOME=/opt/rh/rh-python34/root/usr/
export PYTHONPATH=/opt/rh/rh-python34/root/usr/lib64/python3.4

pip3.4 install --upgrade pip
pip3.4 install requests pyopenssl
pip3.4 install https://github.com/Chaffelson/nipyapi/archive/efm.zip
pip3.4 install asyncio

tee /root/prep_minifi.py <<-'EOF'
#!/usr/bin/python3

import socket
import nipyapi


host_name = socket.getfqdn()
nipyapi.utils.set_endpoint('http://' + host_name + ':61080/nifi-registry-api')

# Create bucket for EFM
try:
    _ = nipyapi.versioning.create_registry_bucket('efm')
except ValueError:
    pass

EOF

chmod +x /root/prep_minifi.py
python3 /root/prep_minifi.py

# i-cpp-0.6.0/bin/minifi: /lib64/liblzma.so.5: version `XZ_5.2.2' not found (required by /etc/minifi-cpp/nifi-minifi-cpp-0.6.0/bin/minifi)

# Start MiNiFi Services
${EFM_HOME}/bin/efm.sh start &> start_efm.log &
${MINIFICPP_HOME}/bin/minifi.sh start &> start_minificpp.log &
${MINIFIJAVA_HOME}/bin/minifi.sh start &> start_minifijava.log &

tee /root/minificpp_template.yaml <<-'EOF'
NiPyAPI Agent Config Version: 1
Processors:
- name: TailLogs
  class: org.apache.nifi.minifi.processors.TailFile
  scheduling period: 1s
  Properties:
    File to Tail: /var/log/messages
    Input Delimiter: \n
Connections:
- name: TailLogsToRouteNiFi
  source: TailLogs
  source relationship names:
  - success
  destination: NIFIHOSTS
  port: fromMiNiFiCpp
Remote Process Groups:
- name: NIFIHOSTS
  transport_protocol: HTTP
EOF

tee /root/minifijava_template.yaml <<-'EOF'
NiPyAPI Agent Config Version: 1
Processors:
- name: ReadMetrics
  class: org.apache.nifi.processors.standard.GenerateFlowFile
  scheduling period: 1 sec
  concurrency: 1
  Properties:
    generate-ff-custom-text: |
      {"__time": "${now():format("yyyy-MM-dd'T'HH:mm:ss'Z'")}","ev_source": "${random():mod(5):toNumber():plus(1)}","ev_metric": "${random():mod(10):toNumber():plus(1)}"}
Connections:
- name: ReadMetricsToRouteNiFi
  source: ReadMetrics
  source relationship names:
  - success
  destination: NIFIHOSTS
  port: fromMiNiFiJava
Remote Process Groups:
- name: NIFIHOSTS
  transport_protocol: HTTP
EOF


##################################################

nifi_host = get_component_host('NIFI', 'NIFI_MASTER')
nifi_hosts = get_component_host('NIFI', 'NIFI_MASTER', all=True)
kafka_broker = get_component_host('KAFKA', 'KAFKA_BROKER')
kafka_brokers = get_component_host('KAFKA', 'KAFKA_BROKER', all=True)
schema_registry_host = get_component_host('REGISTRY', 'REGISTRY_SERVER')
nifi_reg_host = get_component_host('NIFI_REGISTRY', 'NIFI_REGISTRY_MASTER')
zk_hosts = get_component_host('ZOOKEEPER', 'ZOOKEEPER_SERVER', all=True)

brokerlist = ','.join([x + ':' + kafka_port for x in kafka_brokers])
nifilist = ','.join(['http://' + x + ':' + nifi_port + '/nifi' for x in nifi_hosts])
zklist = ','.join([x + ':' + zk_port for x in zk_hosts])

print("NiFi Host: " + nifi_host)
print("Kafka Broker: " + kafka_broker)

echo "Finished deploying and configuring NiFi services"

exit 0