Example of JSON streamer (encoder/decoder) with memory profile

jsonstreamer.py

#!/usr/bin/env python
import sys
from collections import Mapping, Container

import cherrypy
import json
from json import JSONEncoder, JSONDecoder
import cStringIO as StringIO

# code taken from:
# https://code.tutsplus.com/tutorials/understand-how-much-memory-your-python-objects-use--cms-25609
def deep_getsizeof(o, ids):
    """Find the memory footprint of a Python object

    This is a recursive function that drills down a Python object graph
    like a dictionary holding nested dictionaries with lists of lists
    and tuples and sets.

    The sys.getsizeof function does a shallow size of only. It counts each
    object inside a container as pointer only regardless of how big it
    really is.

    :param o: the object
    :param ids:
    :return:
    """
    d = deep_getsizeof
    if id(o) in ids:
        return 0

    r = sys.getsizeof(o)
    ids.add(id(o))

    if isinstance(o, str) or isinstance(0, unicode):
        return r

    if isinstance(o, Mapping):
        return r + sum(d(k, ids) + d(v, ids) for k, v in o.iteritems())

    if isinstance(o, Container):
        return r + sum(d(x, ids) for x in o)

    return r

def jsonstreamer(func):
    """JSON streamer decorator"""
    def wrapper (self, *args, **kwds):
        """Decorator wrapper"""
        cherrypy.response.headers['Content-Type'] = "application/json"
        func._cp_config = {'response.stream': True}
        data = func (self, *args, **kwds)
        if  isinstance(data, dict):
            for chunk in JSONEncoder().iterencode(data):
                yield chunk
        elif  isinstance(data, list) or isinstance(data, types.GeneratorType):
            sep = ''
            for rec in data:
                if  sep:
                    yield sep
                for chunk in JSONEncoder().iterencode(rec):
                    yield chunk
                if  not sep:
                    sep = ', '
        else:
            msg = 'jsonstreamer, improper data type %s' % type(data)
            raise Exception(msg)
    return wrapper

def decoder(istring):
    decoder = JSONDecoder()
    return decoder.decode(istring)

@jsonstreamer
def test(data):
    return data

# complext data structure with nested dicts
# rdict = {"fl":[1,2,3], 'name': 'bla'}
# data = {"foo":1, "nested":[rdict for _ in range(10)]}

data = {"foo":1, "bla":[1,2,3,4,5]}
print('JSON dumps')
print(json.dumps(data))
print('size: {}'.format(deep_getsizeof(data, set())))

# use StringIO stream to hold our data, a la file
print('JSON stream')
output = StringIO.StringIO()
for chunk in test(data):
    output.write(chunk)
# now we have StringIO object we can read from it, the istring is our JSON stream
istring = output.getvalue()
print(istring)
print('size: {}'.format(deep_getsizeof(istring, set())))

# let's decode the JSON stream, we should get back dict object
data = decoder(istring)
if isinstance(data, dict):
    print("decoded output")
    print(json.dumps(data))
    print('size: {}'.format(deep_getsizeof(data, set())))
output.close()