spark-master.sh

spark-master.sh

#!/bin/bash

sudo apt update
sudo apt install -y wget openjdk-11-jdk

wget http://us.mirrors.quenda.co/apache/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz
wget http://mirror.reverse.net/pub/apache/hadoop/common/hadoop-3.1.2/hadoop-3.1.2.tar.gz

tar xvzf spark-2.4.1-bin-hadoop2.7.tgz
tar xvzf hadoop-3.1.2.tar.gz
export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
export HADOOP_HOME=${HOME}/hadoop-3.1.2
export PATH=${HADOOP_HOME}/bin:$PATH
export SPARK_DIST_CLASSPATH=$(hadoop classpath)
export SPARK_HOME=${HOME}/spark-2.4.1-bin-without-hadoop
export PATH=${SPARK_HOME}/bin:$PATH

$SPARK_HOME/sbin/start-master.sh

echo "run 'spark-slave.sh $(hostname)' on all slave nodes"