This is a Python routine to help read many files using multiprocessing. `files` can be a list of file paths and `file_parser_func` can be any function that reads a file and returns a list.

muiltiprocessing_files.py

from multiprocessing import Pool
from tqdm import tqdm
import pandas as pd

# def file_parser_func(fn : str):
#    return pd.read_csv(fn).to_dict('records')
# files = ['a.csv', 'b.csv']

data = []
with Pool(processes=8) as pool:
    for record in tqdm(pool.imap_unordered(file_parser_func, files), 
                       total=len(files)):
        data.extend(record)

df = pd.DataFrame(data)