Skip to content

Instantly share code, notes, and snippets.

@mr-jstraub
Last active March 19, 2016 13:41
Show Gist options
  • Save mr-jstraub/1244e35df11051f8b28e to your computer and use it in GitHub Desktop.
Save mr-jstraub/1244e35df11051f8b28e to your computer and use it in GitHub Desktop.
zeppelin setup HDP 2.3
#!/bin/sh
# Pre-Reqs.: Make sure git and mvn are installed
SPARK_VER=1.5.2
HADOOP_VER=2.7.1
JAVA_HOME=/usr/java/jdk1.8.0_71
HDP_VER=`hdp-select status hadoop-client | sed 's/hadoop-client - \(.*\)/\1/'`
# Get Zeppelin from Git
git clone https://github.com/apache/incubator-zeppelin
cd incubator-zeppelin
# Start compiling
# If you see "invalid target release: 1.7", make sure to set JAVA_HOME
mvn clean install -DskipTests -Dspark.version=$SPARK_VER -Dhadoop.version=$HADOOP_VER -Drat.skip -Pyarn -Phadoop-2.6 -Pspark-1.5 -Ppyspark
# Configure Zeppelin JVM, App Master JVM, and Executor JVMs with hdp.version
cp conf/zeppelin-env.sh.template conf/zeppelin-env.sh
echo "export HADOOP_CONF_DIR=/etc/hadoop/conf" >> conf/zeppelin-env.sh
echo 'export ZEPPELIN_JAVA_OPTS="-Dhdp.version='$HDP_VER'"' >> conf/zeppelin-env.sh
echo "export SPARK_HOME=/usr/hdp/current/spark-client" >> conf/zeppelin-env.sh
echo 'export JAVA_HOME="'$JAVA_HOME'"' >> conf/zeppelin-env.sh
# Copy hive config
cp /etc/hive/conf/hive-site.xml conf/
# hive.metastore.client.connect.retry.delay
sed -i "s/<value>5s<\/value>/<value>5<\/value>/g" conf/hive-site.xml
# hive.metastore.client.socket.timeout
sed -i "s/<value>1800s<\/value>/<value>1800<\/value>/g" conf/hive-site.xml
# Change port to fix Ambari conflict
cp conf/zeppelin-site.xml.template conf/zeppelin-site.xml
sed -i "s/8080/8081/g" conf/zeppelin-site.xml
# Run once & stop to autogenerate conf/interpreter.json:
./bin/zeppelin-daemon.sh start
echo "Zeppelin init in progress ..."
sleep 10s
./bin/zeppelin-daemon.sh stop
# After initial run, edit conf/interpreter.json:
sed -i "s/local\[\*\]/yarn\-client/g" conf/interpreter.json
sed -i '/"master": "yarn-client",/a "spark.driver.extraJavaOptions": "-Dhdp.version='$HDP_VER'",' conf/interpreter.json
sed -i '/"master": "yarn-client",/a "spark.yarn.am.extraJavaOptions": "-Dhdp.version='$HDP_VER'",' conf/interpreter.json
# Restart Zeppelin
./bin/zeppelin-daemon.sh start
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment