Instructions for setting up Luigi server on AWS EC2

luigi_ec2_setup

Note: THIS IS NOT A SHELL SCRIPT


cd /dev/shm
wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
bash Miniconda3-latest-Linux-x86_64.sh


<LOG OUT AND IN>
conda create --name py36 python=3.6
source activate py36
conda install boto3
conda install luigi
sudo yum -y install httpd
sudo service httpd start
sudo groupadd luigi
sudo useradd -g luigi luigi
sudo mkdir /etc/luigi
sudo chown luigi:luigi /etc/luigi
sudo mkdir /var/run/luigi
sudo mkdir /var/log/luigi
sudo mkdir /var/lib/luigi
sudo chown luigi:luigi /var/run/luigi
sudo chown luigi:luigi /var/log/luigi
sudo chown luigi:luigi /var/lib/luigi
export LUIGI_CONFIG_PATH="/home/ec2-user/production/production/config/
<FILL CONFIG FILE AS APPROPRIATE>


sudo su - luigi
cd /dev/shm
bash Miniconda3-latest-Linux-x86_64.sh
<LOG OUT AND IN>
conda create --name py36 python=3.6
source activate py36
conda install luigi
luigid --background --pidfile /var/run/luigi/luigi.pid --logdir /var/log/luigi


<ON EC2:>
open tcp port 8082


<CONFIG:>
see $LUIGI_CONFIG_FILE


<ON SES (using Ireland eu-west-1):>
verify sender and receiver email addresses