Trace

gistfile1.txt

Traceback (most recent call last):
  File "mp.py", line 16, in <module>
    with Pool(processes=1000) as p : 
  File "/usr/lib/python3.8/multiprocessing/context.py", line 119, in Pool
    return Pool(processes, initializer, initargs, maxtasksperchild,
  File "/usr/lib/python3.8/multiprocessing/pool.py", line 212, in __init__
    self._repopulate_pool()
  File "/usr/lib/python3.8/multiprocessing/pool.py", line 303, in _repopulate_pool
    return self._repopulate_pool_static(self._ctx, self.Process,
  File "/usr/lib/python3.8/multiprocessing/pool.py", line 326, in _repopulate_pool_static
    w.start()
  File "/usr/lib/python3.8/multiprocessing/process.py", line 121, in start
    self._popen = self._Popen(self)
  File "/usr/lib/python3.8/multiprocessing/context.py", line 276, in _Popen
    return Popen(process_obj)
  File "/usr/lib/python3.8/multiprocessing/popen_fork.py", line 19, in __init__
    self._launch(process_obj)
  File "/usr/lib/python3.8/multiprocessing/popen_fork.py", line 69, in _launch
    child_r, parent_w = os.pipe()
OSError: [Errno 24] Too many open files

mp.py

import requests
import os
from multiprocessing import Pool

words = ['apple', 'fruit', 'json', 'gorilla', 'fork','final', 'cricket']*100

base_url = 'https://www.google.com/search?q={}'

def make_req(word, session) :
    resp = session.get(base_url.format(word))
    print("process ", os.getpid())
    print(len(resp.content), resp.url)


if __name__ == "__main__":
    with Pool(processes=1000) as p : 
        sess = requests.session()
        sess.config['keep_alive'] = False
        z = [p.apply_async(make_req,(word,sess)) for word in  words]
        print([i.get() for i in  z])