simple batching of a generator

batchit.py

from pprint import pprint
from itertools import chain, islice
from time import sleep
from random import random


def make_req(x):
    return {'id': x}


def generate_requests():
    for x in range(int(random() * 1000)):
        yield make_req(x)


# http://stackoverflow.com/questions/24527006/split-a-generator-into-chunks-without-pre-walking-it
def chunks(iterable, size=20):
    iterator = iter(iterable)
    for first in iterator:
        yield chain([first], islice(iterator, size - 1))


def printer(batches):
    for i, batch in enumerate(batches):
        print ("---------")
        print("Batch {}".format(i))
        if random() <= 0.1:
            print("***** slow write - backpressure")
            sleep(1)
        pprint(list(batch))


printer(chunks(generate_requests()))