kubernetes dask cluster

dask.yaml

apiVersion: v1
kind: Service
metadata:
  name: daskd-scheduler
  labels:
    app: daskd
    role: scheduler
spec:
  ports:
  - port: 8786
    targetPort: 8786
    name: scheduler
  - port: 8787
    targetPort: 8787
    name: bokeh
  - port: 9786
    targetPort: 9786
    name: http
  selector:
    app: daskd
    role: scheduler
  type: LoadBalancer
--- 
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: daskd-scheduler
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: daskd
        role: scheduler
    spec:
      containers:
      - name: scheduler
        image: vascoosx/dask:latest  # CHANGE THIS TO BE YOUR DOCKER HUB IMAGE
        imagePullPolicy: Always
        command: ["/opt/conda/bin/dask-scheduler"]
        resources:
          requests:
            cpu: 1
            memory: 10000Mi # set aside some extra resources for the scheduler
        ports:
        - containerPort: 8786
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: daskd-worker
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: daskd
        role: worker
    spec:
      containers:
      - name: worker
        image: vascoosx/dask:latest  # CHANGE THIS TO BE YOUR DOCKER HUB IMAGE
        imagePullPolicy: Always
        command: ["/bin/bash", "-cx", "/opt/conda/bin/dask-worker $DASKD_SCHEDULER_SERVICE_HOST:$DASKD_SCHEDULER_SERVICE_PORT_SCHEDULER --nthreads 1"]
        resources:
          requests:
            cpu: 1
            memory: 6000Mi

Dockerfile

FROM  continuumio/miniconda3

ENTRYPOINT [ "/bin/bash", "-c" ]
ADD . /deps
WORKDIR /deps
RUN conda install --yes --file req.txt
CMD ["source activate dask && dask-scheduler"]

execute.sh

gcloud container clusters create daskd-cluster \
    --zone asia-east1-a \
    --num-nodes=2 \
    --enable-autoscaling --min-nodes=1 --max-nodes=100 \
    --machine-type=n1-highmem-16
   
gcloud config set container/cluster daskd-cluster
gcloud container clusters get-credentials daskd-cluster --zone asia-east1-a
kubectl create -f dask.yaml
kubectl scale deployment daskd-worker --replicas=20

req.txt

# This file may be used to create an environment using:
# $ conda create --name <env> --file <this file>
# platform: linux-64
bokeh
ca-certificates
certifi
click
cloudpickle
dask=0.15.3
dask-core=0.15.3
distributed=1.19.1
heapdict=1.0.0
intel-openmp=2018.0.0
jinja2=2.9.6
locket=0.2.0
markupsafe=1.0
mkl=2018.0.0
msgpack-python=0.4.8
numpy=1.13.3
openssl=1.0.2l
pandas=0.20.3
partd=0.3.8
pip=9.0.1
psutil=5.3.1
python=3.5.4
python-dateutil=2.6.1
pytz=2017.2
pyyaml=3.12
setuptools=36.5.0
six=1.10.0
sortedcontainers=1.5.7
sqlite=3.20.1
tblib=1.3.2
tk=8.6.7
toolz=0.8.2
tornado=4.5.2
wheel=0.29.0
xz=5.2.3
yaml=0.1.7
zict=0.1.3
zlib=1.2.11

burrowed from https://github.com/hammerlab/dask-distributed-on-kubernetes/