This script exercises the join as well as left, right and full outer join as implemented in Apache Spark. Employee and Department tables were inspired by the examples in wiki's articles on join operation: https://en.wikipedia.org/wiki/Join_(SQL)

jointesting.py

"""
This script exercises the join as well as left, right and full outer join as implemented in
Apache Spark. Employee and Department tables were inspired by the examples in wiki's articles
on join operation: https://en.wikipedia.org/wiki/Join_(SQL)


Employee
    (31, 'Rafferty')
    (33, 'Jones')
    (33, 'Heisenberg')
    (34, 'Robinson')
    (34, 'Smith')
    (None, 'Williams')
Department
    (31, 'Sales')
    (33, 'Engineering')
    (34, 'Clerical')
    (35, 'Marketing')

Employee Join Department
    (31, ('Rafferty', 'Sales'))
    (33, ('Jones', 'Engineering'))
    (33, ('Heisenberg', 'Engineering'))
    (34, ('Robinson', 'Clerical'))
    (34, ('Smith', 'Clerical'))

Employee Left Outer Join Department
    (None, ('Williams', None))
    (31, ('Rafferty', 'Sales'))
    (33, ('Jones', 'Engineering'))
    (33, ('Heisenberg', 'Engineering'))
    (34, ('Robinson', 'Clerical'))
    (34, ('Smith', 'Clerical'))
Employee Right Outer Join Department
    (31, ('Rafferty', 'Sales'))
    (33, ('Jones', 'Engineering'))
    (33, ('Heisenberg', 'Engineering'))
    (34, ('Robinson', 'Clerical'))
    (34, ('Smith', 'Clerical'))
    (35, (None, 'Marketing'))
Employee Full Outer Join Department
    (None, ('Williams', None))
    (31, ('Rafferty', 'Sales'))
    (33, ('Jones', 'Engineering'))
    (33, ('Heisenberg', 'Engineering'))
    (34, ('Robinson', 'Clerical'))
    (34, ('Smith', 'Clerical'))
    (35, (None, 'Marketing'))
"""

from pyspark import SparkConf, SparkContext

conf = SparkConf().setMaster("local[*]").setAppName("JoinTesting")
sc = SparkContext(conf = conf)

sortit = lambda r: sorted(r, key=lambda x: x[0])

def logit(title, array):
    print title
    for r in array:
        print "\t" + str(r)

employee =   [(31,   "Rafferty"),
              (33,   "Jones"),
              (33,   "Heisenberg"),
              (34,   "Robinson"),
              (34,   "Smith"),
              (None, "Williams")]

logit("Employee", employee)

department = [(31,   "Sales"),
              (33,   "Engineering"),
              (34,   "Clerical"),
              (35,   "Marketing")]

logit("Department", department)

e = sc.parallelize(employee)
d = sc.parallelize(department)

results = sortit(e.join(d).collect())

logit("Employee Join Department", results)

results = sortit(e.leftOuterJoin(d).collect())

logit("Employee Left Outer Join Department", results)

results = sortit(e.rightOuterJoin(d).collect())

logit("Employee Right Outer Join Department", results)

results = sortit(e.fullOuterJoin(d).collect())

logit("Employee Full Outer Join Department", results)