traverse_id.py

from concurrent.futures import ThreadPoolExecutor, as_completed
from user_agent import generate_user_agent
import requests
from pyquery import PyQuery as pq


def singal_parse(id):
    url = 'http://www.miaoss.net/reg.php?id={}'.format(id)
    ret = requests.get(url, params={'User-Agent': generate_user_agent()}, timeout=30)
    ret.encoding = 'utf-8'
    d = pq(ret.text)
    return d("[name='tuijian']").val()


def main(end=100000):
    print('开始咯~~~')
    with ThreadPoolExecutor(max_workers=1) as executor, open('result.txt', 'w') as w:
        id_to_email = {executor.submit(singal_parse, id): id for id in range(3115, end)}
        for future in as_completed(id_to_email):
            _id = id_to_email[future]
            try:
                email = future.result()
            except Exception:
                logging.exception('id={}请求出错'.format(_id))
            else:
                print(_id, email)
                w.write('{}:{}\n'.format(_id, email))


if __name__ == '__main__':
    main()