Last active
July 25, 2022 12:43
-
-
Save abajwa-hw/ac43ebfae8e969e627a71d32e028f101 to your computer and use it in GitHub Desktop.
Install latest HDP 2.6 and Nifi (HDF 3.0) and credit card demo on CentOS 6.6
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#Sets up single node HDP+Nifi+credit card demo on CentOS 6.6 | |
#Run script below and then use ~/CreditCardTransactionMonitor/startDemoServices.sh to start demo | |
export ambari_password=${ambari_password:-admin} | |
export host_count=${host_count:-1} #choose number of nodes | |
export ambari_services=${ambari_services:-HDFS HIVE PIG SPARK MAPREDUCE2 TEZ YARN ZOOKEEPER ZEPPELIN NIFI HBASE PHOENIX STORM KAFKA AMBARI_INFRA ATLAS} #AMBARI_METRICS can be added post-install | |
export hdp_ver=${hdp_ver:-2.6} | |
export hdf_mpack="http://public-repo-1.hortonworks.com/HDF/centos6/3.x/updates/3.0.1.1/tars/hdf_ambari_mp/hdf-ambari-mpack-3.0.1.1-5.tar.gz" | |
export vm_name=sandbox | |
export ambari_version=2.5.1.0 | |
#if [[ "$host_count" -eq 1 ]]; then | |
# #update /etc/hosts with sandbox entry pointing to VMs ip, hostname | |
# curl -sSL https://gist.github.com/abajwa-hw/9d7d06b8d0abf705ae311393d2ecdeec/raw | sudo -E sh | |
# echo "Hostname set to: $(hostname -f)" | |
# #setup rc.local to regenerate hosts file on startup and start HDP services | |
# curl -sSL https://gist.github.com/abajwa-hw/4bf004d0fb065d404760eaeebc15e74e/raw | sudo -E sh | |
#fi | |
#remove unneeded repos for some AMIs | |
if [ -f /etc/yum.repos.d/zfs.repo ]; then | |
rm -f /etc/yum.repos.d/zfs.repo | |
fi | |
if [ -f /etc/yum.repos.d/lustre.repo ]; then | |
rm -f /etc/yum.repos.d/lustre.repo | |
fi | |
yum install -y git python-argparse | |
cd | |
git clone https://github.com/seanorama/ambari-bootstrap.git | |
export install_ambari_server=true | |
curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh | |
echo "Waiting 30s for Ambari to come up..." | |
sleep 30 | |
sudo ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar | |
sudo ambari-server install-mpack --verbose --mpack=${hdf_mpack} | |
sudo ambari-server restart | |
sleep 30 | |
echo "Setting recommendation strategy..." | |
export ambari_stack_version=${hdp_ver} | |
if [ "${hdp_ver}" != "2.4" ]; then | |
export recommendation_strategy="ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES" | |
fi | |
cd ~/ambari-bootstrap/deploy/ | |
tee configuration-custom.json > /dev/null << EOF | |
{ | |
"configurations" : { | |
"hdfs-site": { | |
"dfs.replication": "1" | |
}, | |
"nifi-ambari-config": { | |
"nifi.security.encrypt.configuration.password": "StrongPassword" | |
}, | |
"hive-site": { | |
"hive.server2.transport.mode": "binary" | |
} | |
} | |
} | |
EOF | |
echo "Generating BP and deploying cluster..." | |
~/ambari-bootstrap/deploy/deploy-recommended-cluster.bash | |
echo "Waiting for cluster to be installed..." | |
sleep 5 | |
#wait until cluster deployed | |
ambari_pass="${ambari_password}" source ~/ambari-bootstrap/extras/ambari_functions.sh | |
ambari_configs | |
ambari_wait_request_complete 1 | |
sleep 5 | |
#install docker | |
echo "Installing docker..." | |
#rpm -iUvh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm | |
rpm -iUvh https://yum.dockerproject.org/repo/main/centos/6/Packages/docker-engine-1.7.1-1.el6.x86_64.rpm | |
#yum update -y | |
yum -y install docker-engine | |
service docker start | |
chkconfig docker on | |
cd ~ | |
git clone https://github.com/vakshorton/CreditCardTransactionMonitor | |
cd CreditCardTransactionMonitor | |
./install.sh | |
#./startDemoServices.sh |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hi! Do this script must be installed on clean CentOS 6.6 or HDP iso?