Last active
September 13, 2020 14:27
-
-
Save aravindkumarsvg/7c54fed6d91a900a5b35a3432f147d52 to your computer and use it in GitHub Desktop.
Vagrantfile for Apache Spark setup in Ubuntu Xenial 16
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- mode: ruby -*- | |
# vi: set ft=ruby : | |
Vagrant.configure("2") do |config| | |
config.vm.box = "ubuntu/xenial64" | |
config.vm.box_check_update = false | |
# Spark Jobs history | |
config.vm.network "forwarded_port", guest: 4040, host: 4040 | |
# Spark Master | |
config.vm.network "forwarded_port", guest: 8080, host: 8080 | |
config.vm.network "forwarded_port", guest: 8081, host: 8081 | |
config.vm.provider "virtualbox" do |vb| | |
vb.gui = false | |
vb.memory = `grep 'MemTotal' /proc/meminfo | sed -e 's/MemTotal://' -e 's/ kB//'`.to_i / 1024 / 4 | |
vb.cpus = `nproc`.to_i | |
end | |
config.vm.provision "shell", inline: <<-SHELL | |
apt-get update | |
apt-get -y --fix-missing upgrade | |
apt-get -y autoremove | |
apt-get install -y make unzip git | |
# Install Java - 1.8 | |
apt-get update | |
apt-get install -y default-jdk | |
echo -e "\n\n\nJava Successfully Installed!!!!!!!!!!!!!!!!\n\n\n" | |
# Install scala - 2.11.12 | |
which scala | |
if [[ $? != 0 ]]; then | |
cd /home/vagrant | |
wget -c http://scala-lang.org/files/archive/scala-2.11.12.deb | |
dpkg -i scala-2.11.12.deb | |
apt-get update | |
apt-get -y install | |
fi | |
echo -e "\n\n\nScala Successfully Installed!!!!!!!!!!!!!!!!\n\n\n" | |
# Install SBT - 0.13.16 | |
which sbt | |
if [[ $? != 0 ]]; then | |
cd /home/vagrant | |
echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list | |
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 | |
apt-get update | |
apt-get install -y sbt | |
mkdir -p /home/vagrant/.sbt && echo -e "[repositories]\n local\n sbt-releases-repo: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]\n sbt-plugins-repo: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]\n maven-central: https://repo1.maven.org/maven2/\n" >> /home/vagrant/.sbt/repositories | |
chown -R vagrant:vagrant /home/vagrant/.sbt | |
fi | |
echo -e "\n\n\nSBT Successfully Installed!!!!!!!!!!!!!!!!\n\n\n" | |
# Install Giter8 | |
if [[ ! -d /home/vagrant/.conscript/bin ]]; then | |
cd /home/vagrant | |
runuser -l vagrant -c "curl https://raw.githubusercontent.com/foundweekends/conscript/master/setup.sh | sh" | |
runuser -l vagrant -c "/home/vagrant/.conscript/bin/cs foundweekends/giter8" | |
fi | |
echo -e "\n\n\nGiter8 Successfully Installed!!!!!!!!!!!!!!!!\n\n\n" | |
# Install Apache Spark - 2.4.0 | |
if [[ ! -d /usr/local/spark ]]; then | |
wget -c https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz -O /home/vagrant/spark-2.4.0-bin-hadoop2.7.tgz | |
tar -xzf /home/vagrant/spark-2.4.0-bin-hadoop2.7.tgz -C /home/vagrant/ | |
mv /home/vagrant/spark-2.4.0-bin-hadoop2.7 /usr/local/spark | |
echo "export JAVA_HOME=/usr/lib/jvm/default-java" >> /etc/profile | |
echo "export PATH=\$PATH:/usr/local/spark/bin" >> /etc/profile | |
fi | |
echo -e "\n\n\nApache Spark Successfully Installed!!!!!!!!!!!!!!!!\n\n\n" | |
SHELL | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment