Data pipelines setup for vagrant

datapipelines_vagrant.sh

# Setup fifthel 2016 workshop

sudo apt-get update

sudo apt-get install build-essential python3-dev python3-pip postgresql-9.3 postgresql-server-dev-9.3 openjdk-7-jdk openjdk-7-jre git-core

cd 
wget http://d3kbcqa49mib13.cloudfront.net/spark-1.6.1-bin-hadoop2.6.tgz
tar zxvf spark-1.6.1-bin-hadoop2.6.tgz
rm spark-1.6.1-bin-hadoop2.6.tgz

touch ~/.exports
echo "export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-i386" >> ~/.exports
echo "export PATH=/home/vagrant/spark-1.6.1-bin-hadoop2.6/bin:$PATH" >> ~/.exports
echo "export SPARK_HOME=/home/vagrant/spark-1.6.1-bin-hadoop2.6" >> ~/.exports
echo "export PYTHONPATH=\$SPARK_HOME/python:\$SPARK_HOME/python/lib/py4j-0.9-src.zip:/vagrant/fifthel-2016-workshop" >> ~/.exports
echo "export PYSPARK_PYTHON=/usr/bin/python3" >> ~/.exports

echo "source ~/.exports" >> ~/.bashrc

cd /vagrant

git clone https://github.com/unnati-xyz/fifthel-2016-workshop.git

sudo pip3 install -r requirements.txt

echo "do postgres stuff" 

#sudo su - postgres
#psql
#alter user postgres with password 'postgres';
#
#sudo vim /etc/postgresql/9.3/bin/pg_hba.conf
#<change peer to md5>
#
# Create the database from create.sql

echo "run luigi tasks"

# Luigi tasks
# luigi --module luigi_tasks.load_trip_task LoadTripTask --local-scheduler
# luigi --module luigi_tasks.terminal_traffic_task TerminalTrafficTrainTask --local-scheduler

echo "python3 runserver.py"
echo "do curl"
# curl -X POST -H "Content-Type: application/json" -d '{ "data":[ { "hour_slot": 2, "day_of_week": 2, "terminal_code": 123 }, { "hour_slot": 10, "day_of_week": 6, "terminal_code": 65 }, { "hour_slot": 7, "day_of_week": 8, "terminal_code": 10 } ] }' "http://localhost:5000/predictTraffic"