Created
December 31, 2015 16:28
-
-
Save ianthomas/934d89567ea8f53d6fb8 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#install prereqs | |
sudo yum install -y python27-devel.x86_64 | |
sudo yum install -y libpng-devel | |
sudo yum install -y freetype-devel | |
#Installing iPython Notebook | |
if grep isMaster /mnt/var/lib/info/instance.json | grep true; | |
then | |
cd /home/hadoop | |
#install pip | |
sudo pip install virtualenv | |
mkdir IPythonNB | |
cd IPythonNB | |
sudo /usr/local/bin/virtualenv -p /usr/bin/python2.7 venv | |
source venv/bin/activate | |
#Install ipython and dependency | |
sudo /home/hadoop/IPythonNB/venv/bin/pip install --upgrade pip | |
sudo /home/hadoop/IPythonNB/venv/bin/pip install "ipython[notebook]" | |
sudo /home/hadoop/IPythonNB/venv/bin/pip install requests numpy | |
sudo /home/hadoop/IPythonNB/venv/bin/pip install matplotlib | |
sudo /home/hadoop/IPythonNB/venv/bin/pip install nltk | |
sudo /home/hadoop/IPythonNB/venv/bin/pip install mllib | |
sudo /home/hadoop/IPythonNB/venv/bin/pip install boto3 | |
#Create profile | |
ipython profile create default | |
#Run on master /slave based on configuration | |
echo "c = get_config()" > /home/hadoop/.ipython/profile_default/ipython_notebook_config.py | |
echo "c.NotebookApp.ip = '*'" >> /home/hadoop/.ipython/profile_default/ipython_notebook_config.py | |
echo "c.NotebookApp.open_browser = False" >> /home/hadoop/.ipython/profile_default/ipython_notebook_config.py | |
echo "c.NotebookApp.port = 8192" >> /home/hadoop/.ipython/profile_default/ipython_notebook_config.py | |
export IPYTHON_HOME=/home/hadoop/IPythonNB/venv/ | |
export PATH=$PATH:$IPYTHON_HOME/bin | |
export IPYTHON_OPTS="notebook --no-browser --config=/home/hadoop/.ipython/profile_default/ipython_notebook_config.py" | |
#nohup ipython notebook --no-browser > /mnt/var/log/python_notebook.log & | |
/usr/bin/pyspark --packages com.databricks:spark-csv_2.10:1.1.0,com.databricks:spark-avro_2.10:2.0.1 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment