Python utilities function to run multiple processes on your function

multiple_process.py

from tqdm.autonotebook import tqdm
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, wait, as_completed

import multiprocessing
import time


def run_multiple_process(func, params=[], num_cpu=None, verbose=True):

    start_time = time.time()

    max_cpu_count = multiprocessing.cpu_count()

    if num_cpu is not None and num_cpu > max_cpu_count:
        num_cpu = max_cpu_count

    pool = ProcessPoolExecutor(num_cpu)
    futures = []

    results = []
    for param in params:
        futures.append(pool.submit(func, **param))

    # Wait for success
    count_completed = 0
    for proc in tqdm(as_completed(futures), total=len(params)):
        # for proc in as_completed(futures):
        count_completed += 1
        result = proc.result()
        results.append(result)

    elapsed_time = round((time.time() - start_time), 3)

    if verbose:
        print("Success: {} seconds".format(elapsed_time))
    return results


if __name__ == "__main__":

    import random

    def example(x, y):
        time.sleep(random.randint(0, 3))
        return x + y

    params = [
        {"x": 1, "y": 2},
        {"x": 3, "y": 4},
        {"x": 5, "y": 6},
    ]

    results = run_multiple_process(example, params=params)
    print(results)

    # Track result with param_id
    def example2(x, y, param_id):
        time.sleep(random.randint(0, 3))
        return {"param_id": param_id, "result": x + y}

    params = [
        {"x": 1, "y": 2, "param_id": 1},
        {"x": 3, "y": 4, "param_id": 2},
        {"x": 5, "y": 6, "param_id": 3},
    ]

    results = run_multiple_process(example2, params=params)
    print(results)