Deploy trucking demo on single node HDP 2.5 (Centos 6)

setup_trucking_demo.sh

#assuming HDP 2.5 single node already installed with Hbase/Storm/Kafka started

export demo_root=~
export JAVA_HOME=/usr/java/default

cd ${demo_root}

sudo yum -y groupinstall "Development Tools"
sudo yum install -y wget git

echo "setup Python..."
wget http://www.python.org/ftp/python/3.3.2/Python-3.3.2.tar.bz2 -O /var/tmp/Python-3.3.2.tar.bz2
bzip2 -cd /var/tmp/Python-3.3.2.tar.bz2 | tar xvf -
cd Python-3.3.2
./configure
make
sudo make install
sudo ln -s /usr/local/bin/python3 /usr/bin/python3
python3 --version

echo "Setup maven..."
wget http://mirror.cc.columbia.edu/pub/software/apache/maven/maven-3/3.0.5/binaries/apache-maven-3.0.5-bin.tar.gz
sudo tar xzf apache-maven-3.0.5-bin.tar.gz -C /usr/local
cd /usr/local
sudo ln -s apache-maven-3.0.5 maven
export M2_HOME=/usr/local/maven
export PATH=${M2_HOME}/bin:${PATH}
echo 'M2_HOME=/usr/local/maven' >> ~/.bashrc
echo 'M2=$M2_HOME/bin' >> ~/.bashrc
echo 'PATH=$PATH:$M2' >> ~/.bashrc

echo "Setup npm..."
sudo yum install npm --enablerepo=epel -y
sudo npm install npm -g
sudo npm install -g grunt-cli
sudo npm install bower -g


echo "Setup activemq..."
sudo mkdir /opt/activemq
cd /opt/activemq
sudo wget http://archive.apache.org/dist/activemq/apache-activemq/5.9.0/apache-activemq-5.9.0-bin.tar.gz
sudo tar xvzf apache-activemq-*.tar.gz
sudo ln -s apache-activemq-5.9.0 latest


echo "Setup kafka..."
cd /usr/hdp/current/kafka-broker/bin
./kafka-topics.sh --create --zookeeper $(hostname -f):2181 --replication-factor 1 --partition 5 --topic truck_speed_events
./kafka-topics.sh --create --zookeeper $(hostname -f):2181 --replication-factor 1 --partition 5 --topic truck_events
./kafka-topics.sh --zookeeper $(hostname -f):2181 --list

echo "Setup HBase..."
echo "create 'driver_dangerous_events', {NAME=> 'events', VERSIONS=>3}" | hbase shell
echo "create 'driver_dangerous_events_count', {NAME=> 'counters', VERSIONS=>3}" | hbase shell
echo "create 'driver_events', {NAME=> 'allevents', VERSIONS=>3}" | hbase shell

#pull code from sujithas fork for now
cd ${demo_root}
git clone https://github.com/sujithasankuhdp/hdp
#git clone https://github.com/georgevetticaden/hdp

#not needed
#sudo chmod o+x ${demo_root}/hdp ${demo_root}/hdp/reference-apps/ ${demo_root}/hdp/reference-apps/iot-trucking-app ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/config ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/config/dev ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/config/dev/registry

#update configs
update_config () {
  output=`curl -u admin:admin -i -H 'X-Requested-By: ambari'  http://localhost:8080/api/v1/clusters`
  cluster_name=`echo $output | sed -n 's/.*"cluster_name" : "\([^\"]*\)".*/\1/p'`
  export HOST=$(hostname -f)
  

  sed -i "s|\(ambari.cluster.name\)=.*|\1=${cluster_name}|" $1
  sed -i "s|\(ambari.server.url\)=.*|\1=http://${HOST}:8080/|" $1
  sed -i "s|\(hbase.zookeeper.host\)=.*|\1=${HOST}|" $1
  sed -i "s|\(trucking.notification.topic.connection.url\)=.*|\1=tcp://${HOST}:61616|" $1
  sed -i "s|\(trucking.activemq.host\)=.*|\1=${HOST}|" $1
  sed -i "s|/Users/gvetticaden|${HOME}|" $1
}

cd ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-storm-topology/src/main/resources/config/dev/registry/
cp trucking-streaming-hdp-service-config.properties trucking-streaming-hdp-service-config.properties.orig
update_config trucking-streaming-hdp-service-config.properties

cd ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/config/dev/registry/
cp ref-app-hdp-service-config.properties ref-app-hdp-service-config.properties.orig
update_config ref-app-hdp-service-config.properties

#change links on trucking demo webapp home page
#for sandbox, change to: sandbox.hortonworks.com
#otherwise use public IP (required for cloud deployments)
export public_ip=$(curl icanhazip.com)
export hostname=$(hostname -f)

if [ "${hostname}" = "sandbox.hortonworks.com" ]; then
    export webui_host=${hostname}
else
    export webui_host=${public_ip}
fi

cd ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/webappResources/views
cp welcome.html welcome.html.orig
sed -i "s|http://hdf.*\.com|http://${webui_host}|" welcome.html


#install bower
cd ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal
bower install --allow-root


#build hdp-app-utils 
cd ${demo_root}/hdp/app-utils/hdp-app-utils 
mvn clean install -DskipTests=true


#Build iot-trucking-app
cd ${demo_root}/hdp/reference-apps/iot-trucking-app 
mvn clean install -DskipTests=true
# With george's latest code, this step failing with: Failure to find com.hortonworks.registries:schema-registry-serdes:jar:0.1.0-SNAPSHOT
# make sure pom for trucking-web-portal has org.ow2.asm dependencies

cd ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-data-simulator
mvn assembly:assembly


#install latest storm view jar
cd /var/lib/ambari-server/resources/views/
rm -f storm-view-2.*.jar
wget https://hipchat.hortonworks.com/files/1/1907/zF4FiDbf3sMXsjy/storm-view-0.1.0.0.jar
chmod 777 storm-view-0.1.0.0.jar

#Instantiate Storm view
source /root/ambari-bootstrap/extras/ambari_functions.sh
ambari_configs

read -r -d '' body <<EOF
{
  "ViewInstanceInfo": {
    "instance_name": "StormAdmin", "label": "Storm View", "description": "Storm View",
    "visible": true,
    "properties": {
      "storm.host" : "$(hostname -f)",
      "storm.port" : "8744"
    }
  }
}
EOF
${ambari_curl}/views/Storm_Monitoring/versions/0.1.0/instances/StormAdmin -X DELETE
echo "${body}" | ${ambari_curl}/views/Storm_Monitoring/versions/0.1.0/instances/StormAdmin -X POST -d @-

#clear views work dir and restart Ambari
rm -rf work*
ambari-server restart
sleep 15

#update storm jar in storm lib dir
mkdir ${demo_root}/oldjars
mv /usr/hdp/2.5.0.0-1245/storm/lib/log4j*-2.1.jar ${demo_root}/oldjars
cp ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-data-simulator/target/log4j*-2.6.2.jar /usr/hdp/2.5.0.0-1245/storm/lib/


echo "Setup complete, starting demo"

#start activemq
sudo /opt/activemq/latest/bin/activemq start xbean:file:/opt/activemq/latest/conf/activemq.xml

#start webapp on port 8081 
cd ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal
nohup mvn jetty:run -X -Dservice.registry.config.location=${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/config/dev/registry -Dtrucking.activemq.host=$(hostname -f) -Djetty.port=8081 &> ${demo_root}/webapp.log &



#Import Nifi flow, modify hostname references in Nifi flow then start it

#write out script for generating data 
cat << EOF > ~/generate_trucking_events.sh
cd ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-data-simulator/target
nohup java -jar stream-simulator-jar-with-dependencies.jar '' -1 hortonworks.hdp.refapp.trucking.simulator.impl.domain.transport.Truck hortonworks.hdp.refapp.trucking.simulator.impl.collectors.FileEventCollector 1 ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/routes/midwest 500 /tmp/truck-sensor-data/telemetry-device-1.txt 10 'Saint Louis to Tulsa' > nohup-telemetry-device-1.out  &
nohup java -jar stream-simulator-jar-with-dependencies.jar '' -1 hortonworks.hdp.refapp.trucking.simulator.impl.domain.transport.Truck hortonworks.hdp.refapp.trucking.simulator.impl.collectors.FileEventCollector 1 ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/routes/midwest 500 /tmp/truck-sensor-data/telemetry-device-2.txt 11 'Saint Louis to Chicago' > nohup-telemetry-device-2.out  &
nohup java -jar stream-simulator-jar-with-dependencies.jar '' -1 hortonworks.hdp.refapp.trucking.simulator.impl.domain.transport.Truck hortonworks.hdp.refapp.trucking.simulator.impl.collectors.FileEventCollector 1 ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/routes/midwest 500 /tmp/truck-sensor-data/telemetry-device-3.txt 12 'Saint Louis to Memphis' > nohup-telemetry-device-3.out  &
nohup java -jar stream-simulator-jar-with-dependencies.jar '' -1 hortonworks.hdp.refapp.trucking.simulator.impl.domain.transport.Truck hortonworks.hdp.refapp.trucking.simulator.impl.collectors.FileEventCollector 1 ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/routes/midwest 500 /tmp/truck-sensor-data/telemetry-device-4.txt 13 'Des Moines to Chicago' > nohup-telemetry-device-4.out  &
nohup java -jar stream-simulator-jar-with-dependencies.jar '' -1 hortonworks.hdp.refapp.trucking.simulator.impl.domain.transport.Truck hortonworks.hdp.refapp.trucking.simulator.impl.collectors.FileEventCollector 1 ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/routes/midwest 500 /tmp/truck-sensor-data/telemetry-device-5.txt 14 'Joplin to Kansas City' > nohup-telemetry-device-5.out  &
nohup java -jar stream-simulator-jar-with-dependencies.jar '' -1 hortonworks.hdp.refapp.trucking.simulator.impl.domain.transport.Truck hortonworks.hdp.refapp.trucking.simulator.impl.collectors.FileEventCollector 1 ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/routes/midwest 500 /tmp/truck-sensor-data/telemetry-device-6.txt 15 'Memphis to Little Rock' > nohup-telemetry-device-6.out  &
nohup java -jar stream-simulator-jar-with-dependencies.jar '' -1 hortonworks.hdp.refapp.trucking.simulator.impl.domain.transport.Truck hortonworks.hdp.refapp.trucking.simulator.impl.collectors.FileEventCollector 1 ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/routes/midwest 500 /tmp/truck-sensor-data/telemetry-device-7.txt 16 'Peoria to Ceder Rapids' > nohup-telemetry-device-7.out  &
nohup java -jar stream-simulator-jar-with-dependencies.jar '' -1 hortonworks.hdp.refapp.trucking.simulator.impl.domain.transport.Truck hortonworks.hdp.refapp.trucking.simulator.impl.collectors.FileEventCollector 1 ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/routes/midwest 500 /tmp/truck-sensor-data/telemetry-device-8.txt 17 'Springfield to KC Via Columbia' > nohup-telemetry-device-8.out  &
nohup java -jar stream-simulator-jar-with-dependencies.jar '' -1 hortonworks.hdp.refapp.trucking.simulator.impl.domain.transport.Truck hortonworks.hdp.refapp.trucking.simulator.impl.collectors.FileEventCollector 1 ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/routes/midwest 500 /tmp/truck-sensor-data/telemetry-device-9.txt 18 'Springfield to KC Via Hanibal' > nohup-telemetry-device-9.out  &
EOF

chmod +x ~/generate_trucking_events.sh

#append to start_services.sh 
cat <<EOF >> /root/start_services.sh 

echo "Starting activemq..."
sudo /opt/activemq/latest/bin/activemq start xbean:file:/opt/activemq/latest/conf/activemq.xml

echo "Starting webapp on port 8081..."
cd ${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal
nohup /usr/local/maven/bin/mvn jetty:run -X -Dservice.registry.config.location=${demo_root}/hdp/reference-apps/iot-trucking-app/trucking-web-portal/src/main/resources/config/dev/registry -Dtrucking.activemq.host=$(hostname -f) -Djetty.port=8081 &> /var/log/iotdemo-webapp.log &

echo "To start trucking demo: Use webapp (running on port 8081) to deploy topology and view map. Then generate 500 truck events by running ~/generate_trucking_events.sh"
EOF


echo "Trucking demo setup complete"