Large Scale Pytorch Inference Pipeline: Spark vs Dask - Code Examples

emr.Dockerfile

FROM amazoncorretto:8

ENV PYSPARK_DRIVER_PYTHON python3
ENV PYSPARK_PYTHON python3

RUN yum -y update
RUN yum -y groupinstall development
RUN yum -y update \
  && yum -y group install "Development Tools" development \
  && yum -y install yum-utils which hostname python3-devel python-devel python3-pip python3-virtualenv

RUN pip3 install --upgrade pip wheel setuptools
RUN pip3 install pipenv==2018.11.26

WORKDIR /opt/emr-job/
COPY Pipfile.lock .
COPY Pipfile .

RUN pipenv install --system --deploy --ignore-pipfile

COPY . .
RUN python3 setup.py bdist_wheel
RUN python3 setup.py install