pyspark_multithreading.py

import time
from queue import Queue
from threading import Thread

table_list = [
    "table1",
    "table2",
    "table3",
    "table4",
    "table5",
    "table6",
    "table7",
    "table8",
    "table9",
    "table10",
]

q = Queue()
num_of_parallel_tasks = 4
for table in table_list:
    q.put(table)


def load_table(table):
    print(f"Loading {table}")
    time.sleep(10)
    # Write your spark code here
    print(f"{table} loaded")


def run_tasks(function, q):
    while not q.empty():
        value = q.get()
        function(value)
        q.task_done()


for i in range(num_of_parallel_tasks):
    t = Thread(target=run_tasks, args=(load_table, q))
    t.daemon = True
    t.start()

q.join()