gee_on_coiled.py

gee_on_coiled.py

import coiled

cluster = coiled.Cluster(...)

# this will send long-lived Application Default Credentials, with the OAuth scope needed for GEE
coiled.credentials.google.send_application_default_credentials(cluster, scopes=["https://www.googleapis.com/auth/earthengine"])

# plugin to init GEE on each worker 
from distributed.diagnostics.plugin import WorkerPlugin

class InitEarthEngine(WorkerPlugin):
    def __init__(self, **kwargs):
        self.kwargs = kwargs

    def setup(self, worker):
        import ee
        ee.Initialize(**self.kwargs)


# register the plugin
client = cluster.get_client()
client.register_plugin(InitEarthEngine())


# try something really basic, run on each worker to confirm that they're able able to use GEE
import ee

client.run(ee.data.getAlgorithms)