airflow install

airflow_master_scheduler.service

# systemd configuration for master scheduler

[Unit]
Description=Airflow webserver daemon
After=rabbitmq-server.service
Wants=rabbitmq-server.service

[Service]
WorkingDirectory=/opt/airflow
Environment="AIRFLOW_HOME=/opt/airflow"
EnvironmentFile=/opt/airflow/.env
Environment="PATH=/opt/virtualenv/airflow/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
User=lorneluo
Group=lorneluo
Type=simple
ExecStart=/opt/virtualenv/airflow/bin/python /opt/virtualenv/airflow/bin/airflow scheduler --pid /run/airflow/scheduler.pid
Restart=on-failure
RestartSec=5s
PrivateTmp=true

[Install]
WantedBy=multi-user.target

airflow_master_webserver.service

# systemd configuration for master webserver

[Unit]
Description=Airflow webserver daemon
After=network.target rabbitmq-server.service
Wants=rabbitmq-server.service

[Service]
WorkingDirectory=/opt/airflow
EnvironmentFile=/opt/airflow/.env
Environment="PATH=/opt/virtualenv/airflow/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
Environment="AIRFLOW_HOME=/opt/airflow"
#Environment="FLASK_ENV=development"
User=lorneluo
Group=lorneluo
Type=simple
ExecStart=/opt/virtualenv/airflow/bin/python /opt/virtualenv/airflow/bin/airflow webserver --pid /run/airflow/webserver.pid
Restart=on-failure
RestartSec=5s
PrivateTmp=true

[Install]
WantedBy=multi-user.target

airflow_worker.service

# systemd configuration for worker
# replace {EPS-PY-TaskRunner2}，refer https://github.com/lobdev/airflow/blob/master/config/settings.py#L31

[Unit]
Description=Airflow webserver daemon
After=network.target
Wants=

[Service]
# .env example refer https://github.com/lobdev/airflow/blob/master/.env.example
WorkingDirectory=/opt/airflow
Environment="AIRFLOW_HOME=/opt/airflow"
EnvironmentFile=/opt/airflow/.env
Environment="PATH=/opt/airflow/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH"
User=ubuntu
Group=ubuntu
Type=simple
ExecStart=/opt/airflow/venv/bin/airflow worker -q {EPS-PY-TaskRunner2},airflow --pid /run/airflow/worker.pid
Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target

debug.cmd

# check how many task queued on master
sudo rabbitmqctl list_queues --vhost airflow

step1_rabbitmq_install.sh

# install rabbitmq on airflow master
# https://www.rabbitmq.com/install-debian.html
# ubuntu 20.4

# basic ubuntu setup
sudo apt-get install -y curl gnupg apt-transport-https

# Add repositories
echo 'deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang' >> /etc/apt/sources.list.d/bintray.rabbitmq.list
echo 'deb https://dl.bintray.com/rabbitmq/debian bionic main' >> /etc/apt/sources.list.d/bintray.rabbitmq.list

# install Erlang
sudo apt-get update -y
sudo apt-get install -y erlang-base \
                        erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets \
                        erlang-mnesia erlang-os-mon erlang-parsetools erlang-public-key \
                        erlang-runtime-tools erlang-snmp erlang-ssl \
                        erlang-syntax-tools erlang-tftp erlang-tools erlang-xmerl

# install rabbit mq
curl -fsSL https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc | sudo apt-key add -

sudo apt-key adv --keyserver "hkps://keys.openpgp.org" --recv-keys "0x0A9AF2115F4687BD29803A206B73A36E6026DFCA"
sudo apt-get install apt-transport-https
sudo apt-get install rabbitmq-server -y --fix-missing

# configuration
sudo wget https://raw.githubusercontent.com/rabbitmq/rabbitmq-server/master/deps/rabbit/docs/rabbitmq.conf.example -O /etc/rabbitmq/rabbitmq.conf
sudo wget https://raw.githubusercontent.com/rabbitmq/rabbitmq-server/master/deps/rabbit/docs/advanced.config.example -O /etc/rabbitmq/advanced.config

# install rabbitmqadmin
sudo rabbitmq-plugins enable rabbitmq_management
sudo wget http://127.0.0.1:15672/cli/rabbitmqadmin
sudo chmod +x rabbitmqadmin

# create rabbitmq user
sudo rabbitmqctl add_user celery_broker celery_broker_pwd
sudo rabbitmqctl set_user_tags celery_broker administrator
sudo rabbitmqctl add_vhost airflow
sudo rabbitmqctl set_permissions airflow celery_broker ".*" ".*" ".*"

# create queue
./rabbitmqadmin declare queue --username=celery_broker --password=celery_broker_pwd --vhost=airflow name=airflow durable=true

# launch service
sudo service rabbitmq-server start

step2_install_airflow_master.sh

# https://airflow.apache.org/docs/stable/installation.html

sudo apt-get update -y
sudo apt-get install -y build-essential mysql-server libmysqlclient-dev libblas-dev libatlas-base-dev git libffi-dev
sudo apt-get install -y --no-install-recommends \
        freetds-bin \
        krb5-user \
        ldap-utils \
        libffi6 \
        libsasl2-2 \
        libsasl2-modules \
        libssl1.1 \
        locales  \
        lsb-release \
        sasl2-bin \
        sqlite3 \
        unixodbc
        
# install python 3.8.5
# if using pyenv
cd ~/.pyenv
git pull
pyenv install 3.8.5


# work directory & virtualenv
export AIRFLOW_HOME=/opt/airflow
git clone [email protected]:lobdev/airflow.git /opt/airflow
cd /opt/airflow
python3.8 -m venv venv
source ./venv/bin/activate

# install airflow
pip install \
 apache-airflow==1.10.12 \
 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-1.10.12/constraints-3.7.txt"
 
pip install apache-airflow[mysql,celery,rabbitmq,crypto]==1.10.12 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-1.10.12/constraints-3.7.txt"

# configuration
cp .env.template .env
# configure items in .env

# init database
airflow initdb

# nginx install
# https://www.nginx.com/resources/wiki/start/topics/tutorials/install/

# test running
airflow webserver
airflow scheduler

# launch airflow
# official provided some scripts to help to run airflow as daemons https://github.com/apache/airflow/tree/master/scripts
# refer airflow_master_scheduler.service and airflow_master_webserver.service if use systemd

step3_install_airflow_worker.sh

# install airflow worker on each runner
sudo apt-get update -y
sudo apt-get install -y build-essential libmysqlclient-dev libblas-dev libatlas-base-dev git libffi-dev
sudo apt-get install -y --no-install-recommends \
        freetds-bin \
        ldap-utils \
        libffi6 \
        libsasl2-2 \
        libsasl2-modules \
        locales  \
        lsb-release \
        sasl2-bin

# install python 3.8.5
# if using pyenv
cd ~/.pyenv
git pull
pyenv install 3.8.5

# work directory & virtualenv
export AIRFLOW_HOME=/opt/airflow
git clone [email protected]:lobdev/airflow.git /opt/airflow
cd /opt/airflow
python3.8 -m venv venv
source ./venv/bin/activate

# install airflow
pip install \
 apache-airflow==1.10.12 \
 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-1.10.12/constraints-3.7.txt"
 
pip install apache-airflow[mysql,celery,rabbitmq,crypto]==1.10.12 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-1.10.12/constraints-3.7.txt"

# configuration
cp .env.template .env
# configure items in .env

# test running with specified queue name
# queue name should follow https://github.com/lobdev/airflow/blob/master/config/settings.py#L34
airflow worker -q test-queue,airflow

# launch airflow
# official provided some scripts to help to run airflow as daemons https://github.com/apache/airflow/tree/master/scripts
# refer airflow-worker.service if use systemd