Skip to content

Instantly share code, notes, and snippets.

@fgregg
Last active March 4, 2020 21:47
Show Gist options
  • Save fgregg/6f5aa3fe3f9e6142c734a3d9cbff2791 to your computer and use it in GitHub Desktop.
Save fgregg/6f5aa3fe3f9e6142c734a3d9cbff2791 to your computer and use it in GitHub Desktop.
Copy from generator
import psycopg2
import io
import csv
import itertools
class Readable(object):
def __init__(self, iterator):
self.output = io.StringIO()
self.writer = csv.writer(self.output)
self.iterator = iterator
def read(self, size):
self.writer.writerows(itertools.islice(self.iterator, size))
chunk = self.output.getvalue()
self.output.seek(0)
self.output.truncate(0)
return chunk
con = pyscopg2.connect(...)
with con.cursor() as c:
c.copy_expert('COPY your_table FROM STDIN WITH CSV',
Readable(range(10000000000))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment