PySpark example

sparky.py

# ~/spark-3.0xxxx/bin/spark-submit sparky.py
from pyspark.sql import SparkSession

spark = SparkSession.builder.appName("test").getOrCreate()
sc = spark.sparkContext

# Word Count
inputfile = sc.textFile("input.txt")
counts = inputfile.flatMap(lambda x: x.split(" ")).map(lambda x: (x, 1)).reduceByKey(lambda x,y: x+y)
counts.saveAsTextFile("output")

accum = sc.accumulator(0)
def ac(x):
    global accum
    accum += x
sc.parallelize([1,2,3,4]).foreach(ac)
print(accum.value)