Multithreaded file loading example

multithread_fileload.py

# See also https://superfastpython.com/threadpoolexecutor-in-python/

import pathlib
from concurrent.futures import ThreadPoolExecutor, as_completed

# Make a list of all excel files in a directory
path = pathlib.Path('path/to/excel/files')
file_list = path.glob('*.xls*')


# Define a function that should be run in parallel
def some_slow_function(filepath: pathlib.Path):
    # Placeholder, define whatever tasks are needed on the file here
    return filepath.name


# Create a ThreadPool which keeps track of many threads
with ThreadPoolExecutor() as executor:
    # Submit tasks to the threadpool. First argument in submit() is the function to run, second arg is passed as a parameter
    futures = [executor.submit(some_slow_function, excel_file) for excel_file in file_list]
    results = []

    # Append results to a list as each thread finishes
    for future in as_completed(futures):
        results.append(future.result())

print(results)