parallel_apply.py

parallel_apply.py

import pandas as pd
from joblib import Parallel, delayed

data = []
func = None

def process_chunk(chunk_size, num):
    res = data[chunk_size * num: chunk_size * (num + 1)].apply(func)
    return res

def parallel_apply(series, func_to_apply, workers):
    global data
    global func
    data = series
    func = func_to_apply
    total_size = len(series)
    chunk_size = total_size / workers
    result = Parallel(n_jobs=workers, verbose=0)(delayed(process_chunk)(chunk_size, i) for i in range(workers))
    return result[0]
    
# example
def func(x):
    for i in range(1000):
        (x + i) ** 2
    return x ** 5

sample = pd.Series(range(10**4))

res = parallel_apply(sample, func, 4)