abajwa-hw

#!/usr/bin/env bash
# Launch Centos/RHEL 6 with at least 4 cores / 16Gb mem / 60Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/a7d03362ff797c14519128b6c50c26a4/raw | sudo -E sh  


export centos_version=6

export ambari_password=${ambari_password:-StrongPassword}
export db_password=${db_password:-StrongPassword}

abajwa-hw

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<flowController encoding-version="1.3">
  <maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
  <maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>
  <registries/>
  <rootGroup>
    <id>94d19823-0163-1000-97c6-e7c5579e4a0c</id>
    <name>NiFi Flow</name>
    <position x="0.0" y="0.0"/>
    <comment/>

abajwa-hw

# DEPRECATED!!!
# The latest script is at https://github.com/abajwa-hw/masterclass/blob/master/ranger-atlas/
#
#!/usr/bin/env bash
# Launch Centos 7 Vm 
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/440f253831430f7b48d8d6111d92d7b9/raw | sudo -E bash
# Once setup, you can enable LLAP and restart Zeppelin before running sample notebooks
# Known issue: shell interpreter no longer available in Zeppelin in HDP 3.0

abajwa-hw

#onboard N users and HDFS home directories

numusers=5
userprefix="testuser"
group="testusers"
users=()
tables=("hortoniabank.ww_customers" "hortoniabank.us_customers" "finance.tax_2009" "finance.tax_2010" "finance.tax_2015" "cost_savings.claim_savings" "claim.provider_summary" "consent_master.consent_data")
export hive_port=10500

groupadd ${group}

abajwa-hw

#!/usr/bin/env bash
# Launch Centos/RHEL 7 VM with at least 8 vcpu / 32Gb+ memory / 100Gb disk
# Then run:	
# export cluster_name=DESIRED_CLUSTER_NAME ; curl -sSL https://gist.github.com/abajwa-hw/bbe2bdd1ed6a0f738a90dd4e07480e3b/raw | sudo -E sh 

export cluster_name=${cluster_name:-hdp}
export ambari_password=${ambari_password:-BadPass#1}     #Ambari password
export host_count=${host_count:-1}   #choose number of nodes
export ambari_services=${ambari_services:-HDFS HIVE PIG SPARK MAPREDUCE2 TEZ YARN ZOOKEEPER ZEPPELIN NIFI AMBARI_INFRA_SOLR KAFKA SPARK2}  #AMBARI_METRICS can be added post-install
export hdp_ver=${hdp_ver:-3.1}

abajwa-hw

#!/usr/bin/env bash
# Launch Centos/RHEL 7 Vm  with at least 4 cores / 16Gb mem / 60Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/b7c027d9eea9fbd2a2319a21a955df1f/raw | sudo -E sh  

export ambari_password=${ambari_password:-StrongPassword}
export db_password=${db_password:-StrongPassword}
export nifi_password=${nifi_password:-StrongPassword}
export ambari_services="ZOOKEEPER STREAMLINE NIFI KAFKA STORM REGISTRY NIFI_REGISTRY AMBARI_METRICS"
export cluster_name=${cluster_name:-hdf}

abajwa-hw

#!/usr/bin/env bash
# Launch Centos 7 Vm on IBM Power with at least 8 cores / 32Gb mem / 20Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/b0b37d69364430451c3f366558680967/raw | sudo -E sh  


export ambari_version=2.6.0.5
export ambari_repo="http://public-repo-1.hortonworks.com/ambari/centos7-ppc/2.x/updates/2.6.0.5-1/ambari.repo"
export hdp_repo="http://public-repo-1.hortonworks.com/HDP/centos7-ppc/2.x/updates/2.6.3.0/hdp.repo"
export hdp_vdf="http://public-repo-1.hortonworks.com/HDP/centos7-ppc/2.x/updates/2.6.3.0/HDP-2.6.3.0-235.xml"

abajwa-hw

#!/usr/bin/env bash
# Launch Centos 7 Vm on IBM Power with at least 8 cores / 32Gb mem / 20Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/31b656ee03419f23bbcea66a635661b6/raw | sudo -E sh  


export ambari_version=2.6.0.5
export ambari_repo="http://public-repo-1.hortonworks.com/ambari/centos7-ppc/2.x/updates/2.6.0.5-1/ambari.repo"
export hdp_repo="http://public-repo-1.hortonworks.com/HDP/centos7-ppc/2.x/updates/2.6.3.0/hdp.repo"
export hdf_repo="http://public-repo-1.hortonworks.com/HDF/centos7-ppc/3.x/updates/3.0.3.0/hdf.repo"

abajwa-hw

#To run, first copy DMX ambari package to /tmp
#   scp -i ~/.ssh/field.pem ~/Downloads/dmexpress-9.2-el7.ambari-service_en.bin [email protected]:/tmp
#then export any variables and then execute below:
#   curl -sSL https://gist.github.com/abajwa-hw/a4bec6c5fe7545f3515893437746fd38/raw | sudo -E sh 

#For multinode, prep the other hosts first by running below (replace ambari.xxx.com)
#   sudo rpm -Uvh http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm
#   export ambari_version="2.6.1.0";export ambari_server="ambari.xxx.com";curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh

#set -x

abajwa-hw

#install HDP 2.6
export ambari_services="HDFS HIVE PIG SPARK MAPREDUCE2 TEZ YARN ZOOKEEPER ZEPPELIN KAFKA STORM HBASE PHOENIX"
curl -sSL https://gist.github.com/abajwa-hw/7794ea013c96f3f41c4a8b10aeeccd4d/raw | sudo -E sh 


#Setup Mysql
host=$(hostname -f)