threading_webscrapers.py

import threading
import requests

sites = {
   'site1.com': ['https://site1.com/page1' ...],
   'site2.com': ['https://site2.com/page1' ...],
   'site3.com': ['https://site3.com/page1' ...],
   'site4.com': ['https://site4.com/page1' ...]
}

def process(url_list)
    for url in url_list:
        response = requests.get(url)
        do_something_with_response(response)
    
threads = []
for site, url_list in sites:
    thread = threading.Thread(target=process, args=(url_list))  
    threads.append(thread)
    thread.start()

# Makes the program wait for threads to finish
for thread in threads:
    thread.join()