Created
April 7, 2021 06:23
-
-
Save isogram/b2913ac185cb81bea0450f1b74d5b7f6 to your computer and use it in GitHub Desktop.
Python Threading Split in For Loop
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# https://stackoverflow.com/a/15144090/2212582 | |
import threading | |
def process(items, start, end): | |
for item in items[start:end]: | |
try: | |
api.my_operation(item) | |
except Exception: | |
print('error with item') | |
def split_processing(items, num_splits=4): | |
split_size = len(items) // num_splits | |
threads = [] | |
for i in range(num_splits): | |
# determine the indices of the list this thread will handle | |
start = i * split_size | |
# special case on the last chunk to account for uneven splits | |
end = None if i+1 == num_splits else (i+1) * split_size | |
# create the thread | |
threads.append( | |
threading.Thread(target=process, args=(items, start, end))) | |
threads[-1].start() # start the thread we just created | |
# wait for all threads to finish | |
for t in threads: | |
t.join() | |
split_processing(items) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment