jhamman

jhamman

jhamman

MIT License

Copyright (c) 2023 Thomas GAUDIN

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

jhamman

jhamman

from dask.distributed import Client
from dask_gateway import Gateway


gateway = Gateway()  # connect to Gateway

cluster = gateway.new_cluster()  # create cluster
cluster.scale(...)  # scale cluster

client = Client(cluster)  # connect Client to Cluster

jhamman

from dask.distributed import Client
from dask_kubernetes import KubeCluster


cluster = KubeCluster()  # create cluster
cluster.scale(...)  # scale cluster

client = Client(cluster)  # connect Client to Cluster

jhamman

options.worker_cores = 1
options.worker_memory = 8

cluster = gateway.new_cluster(cluster_options=options)

jhamman

containers = [
    {
        "args": [
            "dask-worker", "--nthreads", "1",
            "--memory-limit", "14GB",
        ],
        "resources": {
            "limits": {"cpu": 1, "memory": "14G"},
            "requests": {"cpu": 1, "memory": "14G"},
        },

jhamman

# file: dask_config.yaml
kubernetes:
  worker-template:
    spec:
      containers:
      - args:
          - dask-worker
          - --nthreads
          - '2'
          - --memory-limit

jhamman

$ pip install intake-stac
# or
$ conda install -c conda-forge intake-stac