bootstrap.sh

#!/bin/bash

MASTER=10.0.1.254

export JAVA_HOME=/usr/local/opt/java
mkdir -p $JAVA_HOME
curl -Lb "oraclelicense=a" http://download.oracle.com/otn-pub/java/jdk/7u80-b15/jdk-7u80-linux-x64.tar.gz | tar xz --strip-components=1 -C $JAVA_HOME

export HADOOP_HOME=/usr/local/opt/hadoop
mkdir -p $HADOOP_HOME
curl -L http://www.us.apache.org/dist/hadoop/common/hadoop-2.7.1/hadoop-2.7.1.tar.gz | tar -xz --strip-components=1 -C $HADOOP_HOME

export SPARK_HOME=/usr/local/opt/spark
mkdir -p $SPARK_HOME
curl -L http://www.us.apache.org/dist/spark/spark-1.4.1/spark-1.4.1-bin-hadoop2.6.tgz | tar -xz --strip-components 1 -C $SPARK_HOME

CONF_DIR=/usr/local/etc/hadoop
export HADOOP_CONF_DIR=$CONF_DIR
export SPARK_CONF_DIR=$CONF_DIR
mkdir -p $CONF_DIR

cat << EOF > $CONF_DIR/core-site.xml
<configuration>
<property>
  <name>fs.defaultFS</name>
  <value>hdfs://$MASTER:9000</value>
</property>
</configuration>
EOF

cat << EOF > $CONF_DIR/hdfs-site.xml
<configuration>
<property>
  <name>dfs.replication</name>
  <value>3</value>
</property>
</configuration>
EOF

cat << EOF > $CONF_DIR/yarn-site.xml
<configuration>
<property>
  <name>yarn.nodemanager.aux-services</name>
  <value>mapreduce_shuffle</value>
</property>
<property>
  <name>yarn.resourcemanager.hostname</name>
  <value>$MASTER</value>
</property>
</configuration>
EOF

cat << EOF > $CONF_DIR/capacity-scheduler.xml
<configuration>
<property>
  <name>yarn.scheduler.capacity.root.queues</name>
  <value>default</value>
</property>
<property>
  <name>yarn.scheduler.capacity.root.default.capacity</name>
  <value>100</value>
</property>
</configuration>
EOF

cat << EOF > $CONF_DIR/log4j.properties
log4j.rootCategory=INFO, console
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
log4j.logger.org.eclipse.jetty=WARN
log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=WARN
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=WARN
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=WARN
EOF

cat << EOF > $CONF_DIR/mapred-site.xml
<configuration>
<property>
  <name>mapreduce.framework.name</name>
  <value>yarn</value>
</property>
</configuration>
EOF

cat << EOF > $CONF_DIR/spark-defaults.conf
spark.driver.port                  7001
spark.executor.port                7002
spark.fileserver.port              7003
spark.broadcast.port               7004
spark.replClassServer.port         7005
spark.blockManager.port            7006
EOF

$HADOOP_HOME/bin/hdfs namenode -format
$HADOOP_HOME/bin/hdfs namenode &
$HADOOP_HOME/bin/yarn resourcemanager &