Metron install and Runbook steps

metron_lab.sh

#install HDP 2.6
export ambari_services="HDFS HIVE PIG SPARK MAPREDUCE2 TEZ YARN ZOOKEEPER ZEPPELIN KAFKA STORM HBASE PHOENIX"
curl -sSL https://gist.github.com/abajwa-hw/7794ea013c96f3f41c4a8b10aeeccd4d/raw | sudo -E sh 


#Setup Mysql
host=$(hostname -f)

cat << EOF > mysql-setup.sql
CREATE DATABASE IF NOT EXISTS metronrest;
CREATE USER 'metron'@'$host' IDENTIFIED BY 'StrongPassword';
GRANT ALL PRIVILEGES ON metronrest.* TO 'metron'@'$host' IDENTIFIED BY 'StrongPassword';

use metronrest;
create table if not exists users(
 username varchar(50) not null primary key,
 password varchar(50) not null,
 enabled boolean not null
);
create table authorities (
 username varchar(50) not null,
 authority varchar(50) not null,
 constraint fk_authorities_users foreign key(username) references
 users(username)
);
create unique index ix_auth_username on authorities (username,authority);

use metronrest;
insert into users (username, password, enabled) values ('admin', 'admin',1);
insert into authorities (username, authority) values ('admin', 'ROLE_USER');
flush privileges;
commit; 
EOF

mysql -uroot -p < mysql-setup.sql


#install epel
curl -sL https://rpm.nodesource.com/setup | bash -

ambari-server install-mpack --mpack=https://dev.hortonworks.com.s3.amazonaws.com/HCP/centos7/1.x/BUILDS/1.3.0.0-38/tars/metron/hcp-ambari-mpack-1.3.0.0-38.tar.gz  --verbose
ambari-server restart

# Now install Metron, Elasticsearch, Kibana via Ambari wizard

#Where to install Metron?
#   Pick a node with Storm supervisor:
#   install/start kafka broker
#    pick that node for Metron components


#How many ES? 
#    1 master ES node / 3 data ES nodes



ambari-server install-mpack --mpack=http://public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.0.1.1/tars/hdf_ambari_mp/hdf-ambari-mpack-3.0.1.1-5.tar.gz --verbose
ambari-server restart

# Now install Nifi via Ambari wizard

cd /tmp
git clone https://github.com/crazyadmins/useful-scripts.git
cd useful-scripts/ambari/


#Enable Kerberos

cat << EOF > ambari.props
CLUSTER_NAME=hdp
AMBARI_ADMIN_USER=admin
AMBARI_ADMIN_PASSWORD=admin
AMBARI_HOST=$(hostname -f)
KDC_HOST=$(hostname -f)
REALM=HWX.COM
KERBEROS_CLIENTS=$(hostname -f)
EOF
chmod +x setup_kerberos.sh
./setup_kerberos.sh

kadmin.local -q "modprinc -maxlife 1days -maxrenewlife 7days +allow_renewable krbtgt/[email protected]"
kadmin.local -q "modprinc -maxlife 1days -maxrenewlife 7days +allow_renewable [email protected]"





Lab (Get started with Stellar): https://github.com/cestella/presentations/tree/master/Stellar_tutorial

#Ingest data
Lab (run book - Hbase enrichment)
https://docs.hortonworks.com/HDPDocuments/HCP1/HCP-1.3.0/bk_runbook/content/runbook_overview.html
Use PublishKafka_10 processor instead
sudo usermod -G hadoop,squid nifi
you also have to update kafka broker config on ambari to SASL_PLAINTEXT
Jaas - just paste into Nifi jaas https://community.hortonworks.com/articles/26551/accessing-kerberos-enabled-kafka-topics-using-getk.html
KafkaClient {
    com.sun.security.auth.module.Krb5LoginModule required
    useKeyTab=true
    keyTab="{{nifi_service_keytab}}"
    storeKey=true
    useTicketCache=false
    principal="{{nifi_service_principal}}";
};