Example of using multiprocessing with explicitly batched inputs

multiprocessing_pool_batched.py

import multiprocessing
import itertools

inputs = list(range(111))
batchsize = 10
num_workers = 4

batches = itertools.batched(inputs, batchsize)

def reducer(xs):
    return (sum(xs), len(xs))
with multiprocessing.Pool(processes = num_workers) as pool:
    print(list(pool.map(reducer, batches)))
    
def processor(xs):
    return [x + 1 for x in xs]
with multiprocessing.Pool(processes = num_workers) as pool:
    print(sum(pool.imap_unordered(processor, batches), []))