Testing out performance of the multiprocessing.shared_memory in python 3.8

shared_memory_test.py

from functools import partial
import multiprocessing
from multiprocessing.managers import SharedMemoryManager
import random
import time


NUM_PROCS = 4


def _worker(index, data):
    return data[index] * 2


def _without_shared_memory():
    print("Without shared memory")
    iterations = 6
    for i in range(2, 6):
        indexes = list(range(0, (10 ** i) - 1))
        random.seed(1234)
        random.shuffle(indexes)
        data = list(range(1, 10 ** i))
        chunk_size = len(data) // NUM_PROCS

        func = partial(_worker, data=data)

        start_time = time.time()

        for _ in range(iterations):
            with multiprocessing.Pool(processes=NUM_PROCS) as pool:
                list(pool.imap(func, indexes, chunksize=chunk_size))

        end_time = time.time()
        secs_per_iteration = (end_time - start_time) / iterations
        print(
            "data {0:>10,} ints : {1:>6.6f} secs per iteration".format(
                len(data), secs_per_iteration
            )
        )


def _with_shared_memory():
    print("With shared memory")
    iterations = 6
    for i in range(2, 6):
        with SharedMemoryManager() as smm:
            indexes = list(range(0, (10 ** i) - 1))
            random.seed(1234)
            random.shuffle(indexes)
            data = smm.ShareableList(list(range(1, 10 ** i)))
            chunk_size = len(data) // NUM_PROCS

            func = partial(_worker, data=data)

            start_time = time.time()

            for _ in range(iterations):
                with multiprocessing.Pool(processes=NUM_PROCS) as pool:
                    list(pool.imap(func, indexes, chunksize=chunk_size))

        end_time = time.time()
        secs_per_iteration = (end_time - start_time) / iterations
        print(
            "data {0:>10,} ints : {1:>6.6f} secs per iteration".format(
                len(data), secs_per_iteration
            )
        )


if __name__ == '__main__':
    _without_shared_memory()
    _with_shared_memory()