Easily export a massive dataset from Django to the browser as a streaming CSV file

stream_csv.py

import csv
from StringIO import StringIO
from django.http import StreamingHttpResponse


class StreamCSV(object):

    def __init__(self):
        self._buffer = StringIO()
        self._writer = csv.writer(self._buffer)

    def writerow(self, row):
        """ Stream csv data straight to the connection instead of holding it in memory """
        self._writer.writerow(row)
        self._buffer.seek(0)
        data = self._buffer.read()
        self._buffer.seek(0)
        self._buffer.truncate()
        return data

      
# paginate your query so that data is fetched from the DB in chunks, loop over each chunk and yield
def export(stream):
    yield stream.writerow(['column a', 'column b'])
    for page in some_paginator_object:
        for item in page.object_list:
            yield stream.writerow([item.a, item.b])
      
stream = StreamCSV()
generator = export(stream)
response = StreamingHttpResponse(generator, content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="{}.csv"'.format(download_name)
return response