spark-ray-redis.py

spark-ray-redis.py

import os
import ray
import raydp


HEAD_SERVICE_IP_ENV = "EXAMPLE_CLUSTER_RAY_HEAD_SERVICE_HOST"
head_service_ip = os.environ[HEAD_SERVICE_IP_ENV]

ray.init(address=f"{head_service_ip}:6379")

spark = raydp.init_spark('word_count',
                         num_executors=2,
                         executor_cores=2,
                         executor_memory='1G')

df = spark.createDataFrame([('look',), ('spark',), ('tutorial',), ('spark',), ('look', ), ('python', )], ['word'])
df.show()
word_count = df.groupBy('word').count()
word_count.show()

raydp.stop_spark()