Airflow Docker development environment with different Python versions

README.md

To set up the development environment:

1. docker-compose up -d postgres

2. docker-compose up -d airflow37 # or other version

3. docker logs -f airflow_airflow37_1

4. Wait until you see "Installation ready. Start doing nothing..."

5. docker exec -it airflow_airflow37_1 bash

6. You're good to go, e.g.:

   • airflow webserver
   • airflow scheduler
   • nosetests tests/testfile.py:Class.testname -s

Note: Postgres image only supports one database at a time (without hacking), thus one development environment at a time. So bring only a single Airflow instance up with this script.

docker-compose.yml

version: '3'
services:
  postgres:
    image: postgres:11.2-alpine
    ports:
      - "5432:5432"
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow

# ========================================
# === Default Airflow container config ===
# ========================================
  airflow: &airflow
    image: python
    depends_on:
      - postgres
    environment:
      - AIRFLOW__CORE__EXECUTOR=LocalExecutor
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
      - AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True  # Note! This will fail 'nosetests -s tests/www/test_views.py:TestConfigurationView.test_configuration_do_not_expose_config'. Unset to fix.
      - AIRFLOW__CORE__LOAD_EXAMPLES=True
      - AIRFLOW__CORE__UNIT_TEST_MODE=True
    ports:
      - "8080:8080"
    volumes:
      - $HOME/git/opensource/incubator-airflow:/airflow
    command: /airflow/test-entrypoint.sh
# ========================================

  airflow35:
    <<: *airflow
    image: python:3.5

  airflow36:
    <<: *airflow
    image: python:3.6

  airflow37:
    <<: *airflow
    image: python:3.7

test-entrypoint.sh

#!/usr/bin/env bash

# Prerequisites
apt update
apt install -y libsasl2-dev vim
cd /airflow
pip install --no-use-pep517 -e .[postgres,devel_ci,doc]
pip install ipdb
yes | airflow resetdb

# Create admin user with credentials admin/admin
airflow users --create --username admin --role Admin --email [email protected] --firstname admin --lastname admin --password admin

# Front end stuff
curl -sL https://deb.nodesource.com/setup_11.x | bash -
apt install -y nodejs
cd /airflow/airflow/www
npm install
npm run prod

# Keep container open
echo "Installation ready. Start doing nothing..."
tail -f /dev/null