Airflow MesosExecutor dockerized workflow

airflow.cfg

# Mesos specific block configuration
[mesos]
# Mesos master address which MesosExecutor will connect to.
master = localhost:5050

# The framework name which Airflow scheduler will register itself as on mesos
framework_name = Airflow

# Number of cpu cores required for running one task instance using
# 'airflow run <dag_id> <task_id> <execution_date> --local -p <pickle_id>'
# command on a mesos slave
task_cpu = 1

# Memory in MB required for running one task instance using
# 'airflow run <dag_id> <task_id> <execution_date> --local -p <pickle_id>'
# command on a mesos slave
task_memory = 256

# Enable framework checkpointing for mesos
# See http://mesos.apache.org/documentation/latest/slave-recovery/
checkpoint = False

# Enable framework authentication for mesos
# See http://mesos.apache.org/documentation/latest/configuration/
authenticate = False

# Mesos credentials, if authentication is enabled
# default_principal = admin
# default_secret = admin

# Name of the docker image used for each airflow task
# image created from Dockerfile-airflow build in this case 
docker_image_slave = slicelife/docker-airflow

Dockerfile-airflow

# REFERENCES:
#     - https://github.com/puckel/docker-airflow
#     - https://github.com/ImDarrenG/mesos-framework-dev/blob/master/Dockerfile
#     - https://github.com/Stibbons/docker-airflow-mesos

# Wherever you store your mesos image built from Dockerfile-mesos
FROM slicelife/mesos:1.4.0 as mesos
FROM ubuntu:16.04
# Never prompts the user for choices on installation/configuration of packages
ENV DEBIAN_FRONTEND noninteractive
ENV TERM linux

# Airflow
ARG AIRFLOW_VERSION=1.9.0
ARG AIRFLOW_HOME=/usr/local/airflow

# Define en_US.
ENV LANGUAGE en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
ENV LC_CTYPE en_US.UTF-8
ENV LC_MESSAGES en_US.UTF-8
ENV LC_ALL en_US.UTF-8

# Mesos
ARG MESOS_VERSION=1.4.0
ARG MESOS_HOME=/usr/local/mesos
ARG MESOS_PYTHON_PACKAGES=${MESOS_HOME}/mesos-${MESOS_VERSION}/src/python/dist/

# Copy Mesos python eggs from upstream Mesos image
RUN mkdir -p ${MESOS_PYTHON_PACKAGES}
COPY --from=mesos ${MESOS_PYTHON_PACKAGES} ${MESOS_PYTHON_PACKAGES}

# Install Mesos python eggs
RUN find ${MESOS_HOME} -name "*.egg"
RUN apt-get update -yq --fix-missing
RUN apt-get install -yq \
      python-setuptools \
      build-essential \
      python-dev \
      python-six \
      python-virtualenv \
      libcurl4-nss-dev \
      libsasl2-dev \
      libsasl2-modules \
      maven \
      libapr1-dev \
      libsvn-dev \
      zlib1g-dev
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos.interface-*.egg
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos.executor-*.egg
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos.scheduler-*.egg
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos.native-*.egg
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos.cli-*.egg
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos-*.egg

# Install airflow dependencies
RUN set -ex \
    && buildDeps=' \
        python3-dev \
        libkrb5-dev \
        libsasl2-dev \
        libssl-dev \
        libffi-dev \
        build-essential \
        libblas-dev \
        liblapack-dev \
        libpq-dev \
        git \
    ' \
    && apt-get update -yq \
    && apt-get install -yq --no-install-recommends \
        $buildDeps \
        python3-setuptools \
        python3 \
        python3-pip \
        python3-requests \
        apt-utils \
        curl \
        netcat \
        locales \
    && sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \
    && locale-gen \
    && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \
    && useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow

RUN python3 -m pip install -U pip \
    && pip3 install Cython \
    && pip3 install pytz \
    && pip3 install pyOpenSSL \
    && pip3 install ndg-httpsclient \
    && pip3 install pyasn1 \
    && pip3 install apache-airflow[crypto,postgres,s3,docker,log]==$AIRFLOW_VERSION \
    && pip3 install mesos.cli \
    && pip3 install boto3 \
    && pip3 install flake8 \
    && pip3 install psycopg2-binary \
    && pip3 install pytest \
    && apt-get remove --purge -yq $buildDeps \
    && apt-get clean \
    && rm -rf \
        /var/lib/apt/lists/* \
        /tmp/* \
        /var/tmp/* \
        /usr/share/man \
        /usr/share/doc \
        /usr/share/doc-base

# Add airflow user to root group to properly use the docker.sock
RUN gpasswd -a airflow root

WORKDIR ${AIRFLOW_HOME}

# Install dependencies
COPY requirements.txt .
RUN pip install -r requirements.txt
# Copy over dags, configuration code, and the entrypoint
COPY dags/ dags/
COPY scripts/entrypoint.sh .
COPY config/airflow.cfg .
# Copy over custom logger for s3 logs
COPY config/log_config.py config/log_config.py
COPY config/__init__.py config/__init__.py


# Make all files in ${AIRFLOW_HOME} owned by the airflow user
RUN chown -R airflow: ${AIRFLOW_HOME}
# Adds logging confir on the PYTHONPATH
ENV PYTHONPATH ${PYTHONPATH}:/usr/lib/python3.5/site-packages/:${AIRFLOW_HOME}/config/

EXPOSE 8080

USER airflow
ENTRYPOINT ["./entrypoint.sh"]

Dockerfile-mesos

# This Dockerfile builds Mesos from source, making the python eggs available 
# which is necessary for registering airflow as a framework with Mesos
FROM ubuntu:16.04

# Never prompts the user for choices on installation/configuration of packages
ENV DEBIAN_FRONTEND noninteractive
ENV TERM linux

# Mesos
ARG MESOS_VERSION=1.4.0
ARG MESOS_HOME=/usr/local/mesos

# Install Mesos Dependencies
RUN apt-get update -q --fix-missing
RUN apt-get install -y tar wget git
RUN apt-get install -y openjdk-8-jdk
RUN apt-get -y install build-essential python-dev python-six python-virtualenv libcurl4-nss-dev libsasl2-dev libsasl2-modules maven libapr1-dev libsvn-dev zlib1g-dev

# Install Mesos jar
RUN mkdir -p ${MESOS_HOME}
RUN wget -P ${MESOS_HOME} http://www.apache.org/dist/mesos/${MESOS_VERSION}/mesos-${MESOS_VERSION}.tar.gz
RUN cd ${MESOS_HOME} && tar -zxvf mesos-${MESOS_VERSION}.tar.gz
RUN cd ${MESOS_HOME}/mesos-${MESOS_VERSION} && ./configure && make -j4 V=0 install