Easily export a massive dataset from Django to the browser as a streaming CSV file

stream_csv.py

import csv
from StringIO import StringIO
from django.http import StreamingHttpResponse


class StreamCSV(object):

    def __init__(self):
        self._buffer = StringIO()
        self._writer = csv.writer(self._buffer)

    def writerow(self, row):
        """ Stream csv data straight to the connection instead of holding it in memory """
        self._writer.writerow(row)
        self._buffer.seek(0)
        data = self._buffer.read()
        self._buffer.seek(0)
        self._buffer.truncate()
        return data

      
# paginate your query so that data is fetched from the DB in chunks, loop over each chunk and yield
def export(stream):
    yield stream.writerow(['column a', 'column b'])
    for page in some_paginator_object:
        for item in page.object_list:
            yield stream.writerow([item.a, item.b])
      
stream = StreamCSV()
generator = export(stream)
response = StreamingHttpResponse(generator, content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="{}.csv"'.format(download_name)
return response

I did this: and it worked.

def get_rows_of_responses(template: Template):
    """"""
    # Iterate through all the documents of the template and
    header = []

    def get_expanded_response(document: Document):
        expanded_response = {}
        for key, value in document.symbol_table.items():
            if key in document.response:
                expanded_response[value["name"]] = document.response[key]
            else:
                expanded_response[value["name"]] = ""
        return expanded_response

    def get_expanded_expressions(document: Document):
        expanded_expressions = {}
        for key, value in document.expression_table.items():
            if key in document.expressions:
                expanded_expressions[value["name"]] = document.expressions[key].get(
                    "value"
                )
            else:
                expanded_expressions[value["name"]] = ""
        return expanded_expressions

    documents = Document.objects.filter(template=template).iterator(chunk_size=2)
    for document in documents:
        row = {
            "Document ID": document.id,
            "Document Name": document.name,
            "Created On": document.created_on,
            "Sent?": document.sent,
            "Opened?": document.opened,
            "Completed?": document.completed,
            **get_expanded_response(document),
            **get_expanded_expressions(document),
        }
        if not header:
            header = list(row.keys())
            yield header
        yield [row[key] for key in header]

class TemplateExportResponsesApi(APIView):
    permission_classes = [permissions.IsAuthenticated]

    class Echo:
        """An object that implements just the write method of the file-like
        interface.
        """

        def write(self, value):
            """Write the value by returning it, instead of storing in a buffer."""
            return value

    def get(self, request, template_id):
        template = get_template_by_id(template_id)
        if not can_retrieve_template(user=request.user, template=template):
            raise PermissionDenied()
        rows = get_rows_of_responses(template=template)
        pseudo_buffer = self.Echo()
        writer = csv.writer(pseudo_buffer)
        return StreamingHttpResponse(
            (writer.writerow(row) for row in rows),
            content_type="text/csv",
            headers={
                "Content-Disposition": f'attachment; filename="{template.name}-responses.csv"'
            },
        )