Example Argo Workflow creating a process-based dask.distributed job with a LocalCluster, internal to the pod's node.

dask-internal-process-demo.yaml

apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  generateName: dask-internal-process-demo-
spec:
  entrypoint: dask
  activeDeadlineSeconds: 1800  # Safety first, kids!
  templates:
    - name: dask
      script:
        image: daskdev/dask:2021.12.0
        command: ["python"]
        source: |
          import dask.array as da
          from dask.distributed import Client, LocalCluster

          if __name__ == '__main__':
            cluster = LocalCluster(processes=True, n_workers=4, threads_per_worker=1)
            client = Client(cluster)
          
            array = da.ones((1000, 1000, 1000))
            print(f"OUR ANSWER IS: {array.mean().compute()} (...it should be 1.0 ...)")
        resources:
          requests:
            memory: 16Gi
            cpu: 4000m
          limits:
            memory: 16Gi
            cpu: 5000m