scrape.py

import csv
import redis
import sys
from itertools import zip_longest

r = redis.Redis(host='localhost', port=6379)

csvwriter = csv.writer(sys.stdout)

def batcher(iterable, n):
  args = [iter(iterable)] * n
  return zip_longest(*args)

for keybatch in batcher(r.scan_iter('*'),2000):
  pipe = r.pipeline()
  for key in keybatch:
    if key is not None:
      pipe.type(key)
  types = pipe.execute()
  batchgets = []
  for idx, eachtype in enumerate(types):
    if eachtype == b"string":
      batchgets.append(keybatch[idx])
  pipe = r.pipeline()
  for eachget in batchgets:
    pipe.get(eachget)
  batchget_results = pipe.execute()
  for idx, eachresult in enumerate(batchget_results):
    csvwriter.writerow([batchgets[idx].decode("utf8"), eachresult.decode("utf8")])