Last active
March 16, 2021 22:05
-
-
Save feluelle/7886d695e01ca9095e5916d9240059e6 to your computer and use it in GitHub Desktop.
My Apache Airflow docker-compose file for running LocalExecutor with postgres using official production Dockerfile
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
x-airflow-environment: &airflow-environment | |
environment: | |
- HOST_HOME=${HOME} | |
- HOST_PROJECT_PATH=${PWD} | |
env_file: airflow.env | |
image: feluelle/airflow:latest | |
x-airflow-volumes: &airflow-volumes | |
volumes: | |
- /var/run/docker.sock:/var/run/docker.sock | |
- ~/.aws:/home/airflow/.aws | |
- ../../1-orchestration:/opt/airflow/1-orchestration | |
- ../../2-expectations:/opt/airflow/2-expectations | |
version: '3.8' | |
services: | |
airflow-db: | |
image: library/postgres:latest | |
container_name: airflow_db | |
env_file: postgres.env | |
ports: | |
- 35432:5432 | |
restart: always | |
airflow-db-init: | |
<<: *airflow-environment | |
build: | |
context: https://github.com/apache/airflow.git | |
args: | |
PYTHON_BASE_IMAGE: python:3.8-slim-buster | |
PYTHON_MAJOR_MINOR_VERSION: 3.8 | |
AIRFLOW_EXTRAS: amazon,postgres,google,docker,virtualenv | |
container_name: airflow_db_init | |
command: db upgrade | |
depends_on: | |
- airflow-db | |
airflow-users-init: | |
<<: *airflow-environment | |
container_name: airflow_users_init | |
command: users create --role Admin --username airflow --email [email protected] --firstname Airflow --lastname Apache --password airflow | |
depends_on: | |
- airflow-db-init | |
airflow-webserver: | |
<<: *airflow-environment | |
container_name: airflow_webserver | |
command: webserver | |
ports: | |
- 38080:8080 # airflow webserver | |
# TODO: Remove great_expectations port after moving from python dep to great_expectations docker image | |
- 38888:8888 # great_expectations jupyter notebooks | |
<<: *airflow-volumes | |
depends_on: | |
- airflow-db-init | |
restart: always | |
airflow-scheduler: | |
<<: *airflow-environment | |
container_name: airflow_scheduler | |
command: scheduler | |
<<: *airflow-volumes | |
depends_on: | |
- airflow-db-init | |
restart: always |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
In
airflow.env
I have also the following env variables:The
GE_JUPYTER_CMD
is needed to get great_expecations docs working.