An example of using ijson to process streaming JSON in a memory constrained way

ijson_example.py

#
# This is the core of my handler function
#
batch_size = 100  # This needs to change based on the size of your objects
parser = ijson.parse(f)
counter = 0
to_process = []
for prefix, event, value in parser:
    if (prefix, event) == ("item", "start_map"):
        music_item = dict(albums=[])
    elif (prefix, event) == ("item.album", "start_array"):
        album = dict()
    elif (prefix, event) == ("item.album.item", "start_map"):
        album = dict()
    elif (prefix, event) == ("item.album.item", "end_map"):
        album["id"] = program["album_id"]
        music_item["album"].append(album)
    elif (prefix, event) == ("item.album.item.title", "string"):
        album["title"] = value
    elif (prefix, event) == ("item", "end_map"):
        to_process.append(music_item)
        counter += 1
        if counter > batch_size:
            write_to_mongo(to_process)
            counter = 0
            to_process = []
if to_process:
    write_to_mongo(to_process)
#
# and we're done!
#