Deploy vanilla HDP plus Nifi on single node

deploy_hdp_nifi.sh

#!/usr/bin/env bash
# Launch Centos/RHEL 7 VM with at least 8 vcpu / 32Gb+ memory / 100Gb disk
# Then run:	
# export cluster_name=DESIRED_CLUSTER_NAME ; curl -sSL https://gist.github.com/abajwa-hw/bbe2bdd1ed6a0f738a90dd4e07480e3b/raw | sudo -E sh 

export cluster_name=${cluster_name:-hdp}
export ambari_password=${ambari_password:-BadPass#1}     #Ambari password
export host_count=${host_count:-1}   #choose number of nodes
export ambari_services=${ambari_services:-HDFS HIVE PIG SPARK MAPREDUCE2 TEZ YARN ZOOKEEPER ZEPPELIN NIFI AMBARI_INFRA_SOLR KAFKA SPARK2}  #AMBARI_METRICS can be added post-install
export hdp_ver=${hdp_ver:-3.1}
export nifi_password=${nifi_password:-BadPass#1BadPass#1}
export ambari_version=${ambari_version:-2.7.3.0}
export mpack_url="http://public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.3.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-3.3.0.0-165.tar.gz"


yum install -y git python-argparse mysql-connector-java* nc
cd ~
git clone https://github.com/seanorama/ambari-bootstrap.git

#remove unneeded repos for some AMIs
if [ -f /etc/yum.repos.d/zfs.repo ]; then
  rm -f /etc/yum.repos.d/zfs.repo
fi

if [ -f /etc/yum.repos.d/lustre.repo ]; then
  rm -f /etc/yum.repos.d/lustre.repo
fi  

#install MySql community rpm
sudo rpm -Uvh http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm

export install_ambari_server=true
curl -sSL https://raw.githubusercontent.com/abajwa-hw/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh

while ! echo exit | nc localhost 8080; do echo "waiting for Ambari to be fully up..."; sleep 10; done

echo "Changing Ambari password..." 
curl -iv -u admin:admin -H "X-Requested-By: blah" -X PUT -d "{ \"Users\": { \"user_name\": \"admin\", \"old_password\": \"admin\", \"password\": \"${ambari_password}\" }}" http://localhost:8080/api/v1/users/admin

sudo ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar

echo "Adding HDF mpack..."
sudo ambari-server install-mpack --verbose --mpack=${mpack_url}
sudo ambari-server restart

while ! echo exit | nc localhost 8080; do echo "waiting for Ambari to be fully up..."; sleep 10; done

sleep 20

echo "Setting recommendation strategy..."
export ambari_stack_version=${hdp_ver}
export recommendation_strategy="ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES"

echo "Generating BP and deploying cluster..."
cd ~/ambari-bootstrap/deploy
cat << EOF > configuration-custom.json
{
  "configurations" : {
    "core-site": {
        "hadoop.proxyuser.root.users" : "admin",
        "fs.trash.interval": "4320"
    },
    "yarn-env": {
        "apptimelineserver_heapsize": "1024"
    },    
    "hdfs-site": {
      "dfs.replication": "1",
      "dfs.namenode.safemode.threshold-pct": "0.99"
    },
    "nifi-ambari-config": {
      "nifi.security.encrypt.configuration.password": "${nifi_password}",
      "nifi.sensitive.props.key": "${nifi_password}"
    },    
    "hive-site": {
        "hive.server2.transport.mode" : "binary"
    }
 }
}
EOF

#sed -i.bak "s/\[security\]/\[security\]\nforce_https_protocol=PROTOCOL_TLSv1_2/"   /etc/ambari-agent/conf/ambari-agent.ini
#sudo ambari-agent restart

sleep 30

./deploy-recommended-cluster.bash


##Once deployed run below to enable admin user to use Ambari views
# sudo -u hdfs hdfs dfs -mkdir /user/admin
# sudo -u hdfs hdfs dfs -chown admin /user/admin