Skip to content

Instantly share code, notes, and snippets.

@Anselmoo
Created December 2, 2024 18:11
Show Gist options
  • Save Anselmoo/7b6d737dd811a72ada2e9fbca6adfce6 to your computer and use it in GitHub Desktop.
Save Anselmoo/7b6d737dd811a72ada2e9fbca6adfce6 to your computer and use it in GitHub Desktop.
Threading of str list
import threading
from typing import Callable
def batch_process_str_list(
str_list: list[str],
process_function: Callable[[str, list], None],
batch_size: int = 10,
) -> tuple[list, list]:
results = []
nouns_list = []
for i in range(0, len(str_list), batch_size):
batch = str_list[i : i + batch_size]
threads = []
batch_results = []
for link in batch:
t = threading.Thread(target=process_function, args=(link, batch_results))
t.start()
threads.append(t)
for t in threads:
t.join()
results.extend(batch_results)
return results, nouns_list
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment