Last active
March 19, 2016 13:41
-
-
Save mr-jstraub/1244e35df11051f8b28e to your computer and use it in GitHub Desktop.
zeppelin setup HDP 2.3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
# Pre-Reqs.: Make sure git and mvn are installed | |
SPARK_VER=1.5.2 | |
HADOOP_VER=2.7.1 | |
JAVA_HOME=/usr/java/jdk1.8.0_71 | |
HDP_VER=`hdp-select status hadoop-client | sed 's/hadoop-client - \(.*\)/\1/'` | |
# Get Zeppelin from Git | |
git clone https://github.com/apache/incubator-zeppelin | |
cd incubator-zeppelin | |
# Start compiling | |
# If you see "invalid target release: 1.7", make sure to set JAVA_HOME | |
mvn clean install -DskipTests -Dspark.version=$SPARK_VER -Dhadoop.version=$HADOOP_VER -Drat.skip -Pyarn -Phadoop-2.6 -Pspark-1.5 -Ppyspark | |
# Configure Zeppelin JVM, App Master JVM, and Executor JVMs with hdp.version | |
cp conf/zeppelin-env.sh.template conf/zeppelin-env.sh | |
echo "export HADOOP_CONF_DIR=/etc/hadoop/conf" >> conf/zeppelin-env.sh | |
echo 'export ZEPPELIN_JAVA_OPTS="-Dhdp.version='$HDP_VER'"' >> conf/zeppelin-env.sh | |
echo "export SPARK_HOME=/usr/hdp/current/spark-client" >> conf/zeppelin-env.sh | |
echo 'export JAVA_HOME="'$JAVA_HOME'"' >> conf/zeppelin-env.sh | |
# Copy hive config | |
cp /etc/hive/conf/hive-site.xml conf/ | |
# hive.metastore.client.connect.retry.delay | |
sed -i "s/<value>5s<\/value>/<value>5<\/value>/g" conf/hive-site.xml | |
# hive.metastore.client.socket.timeout | |
sed -i "s/<value>1800s<\/value>/<value>1800<\/value>/g" conf/hive-site.xml | |
# Change port to fix Ambari conflict | |
cp conf/zeppelin-site.xml.template conf/zeppelin-site.xml | |
sed -i "s/8080/8081/g" conf/zeppelin-site.xml | |
# Run once & stop to autogenerate conf/interpreter.json: | |
./bin/zeppelin-daemon.sh start | |
echo "Zeppelin init in progress ..." | |
sleep 10s | |
./bin/zeppelin-daemon.sh stop | |
# After initial run, edit conf/interpreter.json: | |
sed -i "s/local\[\*\]/yarn\-client/g" conf/interpreter.json | |
sed -i '/"master": "yarn-client",/a "spark.driver.extraJavaOptions": "-Dhdp.version='$HDP_VER'",' conf/interpreter.json | |
sed -i '/"master": "yarn-client",/a "spark.yarn.am.extraJavaOptions": "-Dhdp.version='$HDP_VER'",' conf/interpreter.json | |
# Restart Zeppelin | |
./bin/zeppelin-daemon.sh start |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment