Multiprocessing pandas with tqdm process_map

pandas_multiprocessing.py

from tqdm.contrib.concurrent import process_map

# ONLY WORKING ON UBUNTU NOT ON WINDOWS - tested on Python 3.9 - 3.11

# import your pandas df here 
# test_array = df["your_colum"].to_list() # to_list is crucial as it must be picklable

def test_function(x):
    return x*x

test_array = list(range(2_000_000))

processed_items_in_correct_order = process_map(test_function, test_array, chunksize=1_000, max_workers=32) # default max_workers: min(32, cpu_count() + 4)
# docs: https://tqdm.github.io/docs/contrib.concurrent/

# df["new_processed_column"]  = processed_items_in_correct_order # can be done as processed_map always returns the items in order!

processed_items_in_correct_order[:10]
# [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]