Taken from Stephanie Wang's gist: https://gist.github.com/stephanie-wang/7400fd3b50bef0c8ce2d5b29f58c2530#file-pyspark_on_ray-py

pyspark_on_ray.py

import ray
import raydp

ray.init()

@ray.remote
class PySparkDriver:
    def __init__(self):
        self.spark = raydp.init_spark(
            app_name='RayDP example',
            num_executors=2,
            executor_cores=2,
            executor_memory='4GB')

    def foo(self):
        return self.spark.range(1000).repartition(10).count()

driver = PySparkDriver.remote()
print(ray.get(driver.foo.remote()))