Map reduce engine emulator

map_reduce.py

__author__ = 'uolter'


"""
Defines a single function, map_reduce, which takes an input
dictionary i and applies the user-defined function mapper to each
(input_key,input_value) pair, producing a list of intermediate
keys and intermediate values.  Repeated intermediate keys then
have their values grouped into a list, and the user-defined
function reducer is applied to the intermediate key and list of
intermediate values.  The results are returned as a list.
"""


import itertools

def map_reduce(i, mapper, reducer, debug=False):
    intermediate = []
    for (key, value) in i.items():
        map_out = mapper(key, value)
        if debug:
            print(map_out)
        intermediate.extend(map_out)

    groups = {}

    for key, group in itertools.groupby(sorted(intermediate), lambda x: x[0]):
        groups[key] = list([y for x, y in group])

    return [reducer(intermediate_key, groups[intermediate_key]) for intermediate_key in groups]