Skip to content

Instantly share code, notes, and snippets.

@ianthomas
Created December 31, 2015 16:28
Show Gist options
  • Save ianthomas/934d89567ea8f53d6fb8 to your computer and use it in GitHub Desktop.
Save ianthomas/934d89567ea8f53d6fb8 to your computer and use it in GitHub Desktop.
#install prereqs
sudo yum install -y python27-devel.x86_64
sudo yum install -y libpng-devel
sudo yum install -y freetype-devel
#Installing iPython Notebook
if grep isMaster /mnt/var/lib/info/instance.json | grep true;
then
cd /home/hadoop
#install pip
sudo pip install virtualenv
mkdir IPythonNB
cd IPythonNB
sudo /usr/local/bin/virtualenv -p /usr/bin/python2.7 venv
source venv/bin/activate
#Install ipython and dependency
sudo /home/hadoop/IPythonNB/venv/bin/pip install --upgrade pip
sudo /home/hadoop/IPythonNB/venv/bin/pip install "ipython[notebook]"
sudo /home/hadoop/IPythonNB/venv/bin/pip install requests numpy
sudo /home/hadoop/IPythonNB/venv/bin/pip install matplotlib
sudo /home/hadoop/IPythonNB/venv/bin/pip install nltk
sudo /home/hadoop/IPythonNB/venv/bin/pip install mllib
sudo /home/hadoop/IPythonNB/venv/bin/pip install boto3
#Create profile
ipython profile create default
#Run on master /slave based on configuration
echo "c = get_config()" > /home/hadoop/.ipython/profile_default/ipython_notebook_config.py
echo "c.NotebookApp.ip = '*'" >> /home/hadoop/.ipython/profile_default/ipython_notebook_config.py
echo "c.NotebookApp.open_browser = False" >> /home/hadoop/.ipython/profile_default/ipython_notebook_config.py
echo "c.NotebookApp.port = 8192" >> /home/hadoop/.ipython/profile_default/ipython_notebook_config.py
export IPYTHON_HOME=/home/hadoop/IPythonNB/venv/
export PATH=$PATH:$IPYTHON_HOME/bin
export IPYTHON_OPTS="notebook --no-browser --config=/home/hadoop/.ipython/profile_default/ipython_notebook_config.py"
#nohup ipython notebook --no-browser > /mnt/var/log/python_notebook.log &
/usr/bin/pyspark --packages com.databricks:spark-csv_2.10:1.1.0,com.databricks:spark-avro_2.10:2.0.1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment